{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "11abe3e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/cuda117/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([   1,    1,    0, 9178,   32,   47]),\n",
       " tensor([0, 0, 1, 1, 1, 1]),\n",
       " '<pad><pad><s>how are you')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "from util import TokenizerUtil\n",
    "\n",
    "tokenizer = TokenizerUtil()\n",
    "\n",
    "input_ids, _ = tokenizer.encode('how are you', max_length=6)\n",
    "\n",
    "input_ids, attention_mask = tokenizer.pad_to_left(input_ids)\n",
    "\n",
    "input_ids, attention_mask, tokenizer.decode(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "835db30b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7144,\n",
       " {'input_ids': tensor([[   1,    1,    1,  ...,  116, 6267,   35],\n",
       "          [   1,    1,    1,  ...,  116, 6267,   35],\n",
       "          [   1,    1,    1,  ...,  116, 6267,   35],\n",
       "          [   1,    1,    1,  ...,  116, 6267,   35]]),\n",
       "  'attention_mask': tensor([[0, 0, 0,  ..., 1, 1, 1],\n",
       "          [0, 0, 0,  ..., 1, 1, 1],\n",
       "          [0, 0, 0,  ..., 1, 1, 1],\n",
       "          [0, 0, 0,  ..., 1, 1, 1]])})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import default_data_collator\n",
    "\n",
    "dataset = load_dataset('json', data_files='dataset/train.json', split='train')\n",
    "\n",
    "#2,4,4切分,取最后一部分\n",
    "dataset = dataset.select(range(45000, len(dataset)))\n",
    "\n",
    "\n",
    "def f(data):\n",
    "    input_ids, _ = tokenizer.encode(data['prompt'], max_length=256)\n",
    "    input_ids, attention_mask = tokenizer.pad_to_left(input_ids)\n",
    "\n",
    "    return {'input_ids': input_ids, 'attention_mask': attention_mask}\n",
    "\n",
    "\n",
    "dataset = dataset.map(f, remove_columns=dataset.column_names)\n",
    "\n",
    "loader = torch.utils.data.DataLoader(dataset,\n",
    "                                     collate_fn=default_data_collator,\n",
    "                                     batch_size=4,\n",
    "                                     shuffle=True,\n",
    "                                     drop_last=True)\n",
    "\n",
    "len(loader), next(iter(loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5e7480ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:22<00:00, 11.13s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'count_require': 2.21044736, 'count_all': 14.29004288, 'ratio': 0.15468444556549854}\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoConfig, AutoModelForCausalLM, get_scheduler\n",
    "import lora\n",
    "\n",
    "model_actor = AutoModelForCausalLM.from_pretrained('model/actor', dropout=0.0)\n",
    "lora.insert(model_actor)\n",
    "\n",
    "\n",
    "def f():\n",
    "    params = []\n",
    "    params_lora = []\n",
    "    for n, p in model_actor.named_parameters():\n",
    "        if not p.requires_grad:\n",
    "            continue\n",
    "        if 'lora_A' in n or 'lora_B' in n:\n",
    "            params_lora.append(p)\n",
    "            continue\n",
    "        params.append(p)\n",
    "\n",
    "    return [{\n",
    "        'params': params,\n",
    "        'weight_decay': 0.0\n",
    "    }, {\n",
    "        'params': params_lora,\n",
    "        'weight_decay': 0.0,\n",
    "        'lr': 5e-4\n",
    "    }]\n",
    "\n",
    "\n",
    "optimizer_actor = torch.optim.Adam(f(), lr=1e-5, betas=(0.9, 0.95))\n",
    "\n",
    "scheduler_actor = get_scheduler(name='cosine',\n",
    "                                optimizer=optimizer_actor,\n",
    "                                num_warmup_steps=100,\n",
    "                                num_training_steps=800)\n",
    "\n",
    "model_actor.gradient_checkpointing_enable()\n",
    "model_actor.train()\n",
    "\n",
    "lora.count_params(model_actor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bdd68dee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'count_require': 3.31196928, 'count_all': 3.31196928, 'ratio': 1.0}\n"
     ]
    }
   ],
   "source": [
    "class CriticModel(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        from transformers import AutoModel\n",
    "        self.rwtransformer = AutoModel.from_pretrained('facebook/opt-350m',\n",
    "                                                       dropout=0.0)\n",
    "\n",
    "        self.v_head = torch.nn.Linear(512, 1, bias=False)\n",
    "\n",
    "    def get_value(self, input_ids, attention_mask):\n",
    "        value = self.rwtransformer(\n",
    "            input_ids=input_ids,\n",
    "            attention_mask=attention_mask).last_hidden_state\n",
    "        return self.v_head(value).squeeze(2)\n",
    "\n",
    "    def get_reward(self, input_ids, attention_mask):\n",
    "        value = self.get_value(input_ids, attention_mask)\n",
    "\n",
    "        reward = []\n",
    "        for i, v in zip(input_ids, value):\n",
    "            end = input_ids.shape[1] - 1\n",
    "            if tokenizer.eos_token_id in i:\n",
    "                end = i.tolist().index(tokenizer.eos_token_id)\n",
    "            reward.append(v[end])\n",
    "        reward = torch.stack(reward)\n",
    "\n",
    "        return reward\n",
    "\n",
    "\n",
    "model_critic = torch.load('model/critic')\n",
    "\n",
    "optimizer_critic = torch.optim.Adam(model_critic.parameters(),\n",
    "                                    lr=5e-6,\n",
    "                                    betas=(0.9, 0.95))\n",
    "\n",
    "scheduler_critic = get_scheduler(name='cosine',\n",
    "                                 optimizer=optimizer_critic,\n",
    "                                 num_warmup_steps=100,\n",
    "                                 num_training_steps=800)\n",
    "\n",
    "model_critic.train()\n",
    "\n",
    "lora.count_params(model_critic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "17b9d356",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:08<00:00,  4.08s/it]\n",
      "Detected kernel version 3.10.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    }
   ],
   "source": [
    "from accelerate import Accelerator\n",
    "\n",
    "model_ref = AutoModelForCausalLM.from_pretrained('model/actor')\n",
    "model_reward = torch.load('model/critic')\n",
    "\n",
    "model_ref.eval()\n",
    "model_reward.eval()\n",
    "\n",
    "accelerator = Accelerator(gradient_accumulation_steps=1,\n",
    "                          mixed_precision='fp16')\n",
    "\n",
    "(loader, model_actor, optimizer_actor, scheduler_actor, model_critic,\n",
    " optimizer_critic,\n",
    " scheduler_critic) = accelerator.prepare(loader, model_actor, optimizer_actor,\n",
    "                                         scheduler_actor, model_critic,\n",
    "                                         optimizer_critic, scheduler_critic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "787e283d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/root/miniconda3/envs/cuda117/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 299])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@torch.no_grad()\n",
    "def get_generate(input_ids, attention_mask):\n",
    "    generate = model_actor.generate(input_ids,\n",
    "                                    attention_mask=attention_mask,\n",
    "                                    max_length=512,\n",
    "                                    pad_token_id=tokenizer.pad_token_id,\n",
    "                                    eos_token_id=tokenizer.eos_token_id)\n",
    "\n",
    "    lens = (generate[:, 256:] != tokenizer.pad_token_id).sum(1)\n",
    "\n",
    "    return generate[lens > 1]\n",
    "\n",
    "\n",
    "data = next(iter(loader))\n",
    "\n",
    "get_generate(**data).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8687059d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 123])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_prob(prob, index):\n",
    "    prob = prob.log_softmax(dim=2)\n",
    "    prob = prob.gather(dim=2, index=index.unsqueeze(2))\n",
    "    return prob.squeeze(2)\n",
    "\n",
    "\n",
    "get_prob(torch.randn(4, 123, 999), torch.randint(0, 999, (4, 123))).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4d7d1d4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([4, 299]),\n",
       " torch.Size([4, 299]),\n",
       " torch.Size([4, 298]),\n",
       " torch.Size([4, 298]),\n",
       " torch.Size([4, 298]),\n",
       " torch.Size([4]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_generate = None\n",
    "\n",
    "\n",
    "@torch.no_grad()\n",
    "def get_batch(input_ids, attention_mask):\n",
    "    #input_ids -> [4, 256]\n",
    "    #attention_mask -> [4, 256]\n",
    "    global last_generate\n",
    "\n",
    "    #根据问题生成回答\n",
    "    #[4, gen_lens]\n",
    "    generate = get_generate(input_ids, attention_mask)\n",
    "\n",
    "    #制作缓存,防止所有回答为空的情况\n",
    "    if len(generate):\n",
    "        last_generate = generate\n",
    "    else:\n",
    "        generate = last_generate\n",
    "\n",
    "    #[4, gen_lens]\n",
    "    generate_mask = (generate != tokenizer.pad_token_id).long()\n",
    "\n",
    "    #两个模型分别取回答被预测到的概率\n",
    "    #[4, gen_lens-1]\n",
    "    prob_old = model_actor(input_ids=generate,\n",
    "                           attention_mask=generate_mask).logits\n",
    "    prob_old = get_prob(prob_old[:, :-1], generate[:, 1:])\n",
    "\n",
    "    #取每个词的value\n",
    "    #[4, gen_lens-1]\n",
    "    value_old = model_critic.get_value(generate, generate_mask)[:, :-1]\n",
    "\n",
    "    #[4, gen_lens-1]\n",
    "    prob_ref = model_ref(\n",
    "        input_ids=generate.to('cpu'),\n",
    "        attention_mask=generate_mask.to('cpu')).logits.to('cuda')\n",
    "    prob_ref = get_prob(prob_ref[:, :-1], generate[:, 1:])\n",
    "\n",
    "    #取回答的分数\n",
    "    #[4]\n",
    "    reward = model_reward.get_reward(generate.to('cpu'),\n",
    "                                     generate_mask.to('cpu')).to('cuda')\n",
    "\n",
    "    return generate, generate_mask, prob_old, prob_ref, value_old, reward\n",
    "\n",
    "\n",
    "generate, generate_mask, prob_old, prob_ref, value_old, reward = get_batch(\n",
    "    **data)\n",
    "\n",
    "generate.shape, generate_mask.shape, prob_old.shape, prob_ref.shape, value_old.shape, reward.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fb227950",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 298])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_reward_kl(end, prob_old, prob_ref, reward):\n",
    "    #prob_old -> [4, gen_lens-1]\n",
    "    #prob_ref -> [4, gen_lens-1]\n",
    "    #reward -> [4]\n",
    "\n",
    "    #两份预测概率求kl散度\n",
    "    #[4, gen_lens-1]\n",
    "    reward_kl = -0.1 * (prob_old - prob_ref)\n",
    "\n",
    "    #把原本的reward加在kl散度的最后一个字上\n",
    "    for i, e in enumerate(end):\n",
    "        if e >= reward_kl.shape[1]:\n",
    "            e = -1\n",
    "        reward_kl[i, e] += reward[i].clamp(-5, 5)\n",
    "\n",
    "    #[4, gen_lens-1]\n",
    "    return reward_kl\n",
    "\n",
    "\n",
    "end = generate_mask[:, 256:].sum(1) + 255\n",
    "end = end.tolist()\n",
    "\n",
    "reward_kl = get_reward_kl(end, prob_old, prob_ref, reward)\n",
    "\n",
    "reward_kl.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cf23aae7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 43])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#解释见get_delta_note函数\n",
    "def get_delta(value_old, reward_kl):\n",
    "    #value_old -> [4, gen_lens-1]\n",
    "    #reward_kl -> [4, gen_lens-1]\n",
    "\n",
    "    #gen_lens-2 -> 255\n",
    "    delta = []\n",
    "    for i in reversed(range(255, value_old.shape[1])):\n",
    "        #[4]\n",
    "        value_next = 0.0\n",
    "        if i != value_old.shape[1] - 1:\n",
    "            value_next = value_old[:, i + 1]\n",
    "\n",
    "        #[4]\n",
    "        d = reward_kl[:, i] + value_next - value_old[:, i]\n",
    "        if len(delta):\n",
    "            d += 0.95 * delta[-1]\n",
    "        delta.append(d)\n",
    "\n",
    "    #[4, gen_lens-256]\n",
    "    delta = torch.stack(delta[::-1], dim=1)\n",
    "\n",
    "    return delta\n",
    "\n",
    "\n",
    "delta = get_delta(value_old, reward_kl)\n",
    "\n",
    "delta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b1716c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'test success'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get_delta函数的原理解释,注释性代码\n",
    "#数学上和get_delta函数等价,但是运行效率低\n",
    "def get_delta_note(value_old, reward_kl):\n",
    "    #循环中自减会出问题,所以先clone一份再操作\n",
    "    clone = value_old.clone()\n",
    "\n",
    "    #下一个词的value,减去当前词的value,相当于对value去基线,缩小数值方差\n",
    "    #每个词的value是相互独立的,前后词value的差,可以视为预测质量的衡量\n",
    "    for i in range(255, value_old.shape[1]):\n",
    "        value_next = 0.0\n",
    "        if i != value_old.shape[1] - 1:\n",
    "            value_next = value_old[:, i + 1]\n",
    "        clone[:, i] = value_next - value_old[:, i]\n",
    "    value_old = clone\n",
    "\n",
    "    #在value中融合reward,kl\n",
    "    value_old += reward_kl\n",
    "\n",
    "    #蒙特卡洛采样法估计Q函数\n",
    "    #这里计算的其实就是adv\n",
    "    delta = []\n",
    "    for i in range(255, value_old.shape[1]):\n",
    "        s = 0\n",
    "        for j in range(i, value_old.shape[1]):\n",
    "            s += value_old[:, j] * 0.95**(j - i)\n",
    "        delta.append(s)\n",
    "\n",
    "    return torch.stack(delta, dim=1)\n",
    "\n",
    "\n",
    "#测试两个函数是等价的,误差是由于计算机精度导致的\n",
    "for i in range(1000):\n",
    "    value_old_test = torch.randn(4, 285)\n",
    "    reward_kl_test = torch.randn(4, 285)\n",
    "\n",
    "    diff = get_delta(value_old_test, reward_kl_test) - get_delta_note(\n",
    "        value_old_test, reward_kl_test)\n",
    "    diff = diff.abs().max().item()\n",
    "    assert diff < 1e-5\n",
    "'test success'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9cf1bcf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.2290, device='cuda:0')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_actor(prob_new, prob_old, delta, generate_mask):\n",
    "    prob_new = prob_new[:, 255:]\n",
    "    prob_old = prob_old[:, 255:]\n",
    "    generate_mask = generate_mask[:, 256:]\n",
    "\n",
    "    #prob_new -> [4, gen_lens-256]\n",
    "    #prob_old -> [4, gen_lens-256]\n",
    "    #delta -> [4, gen_lens-256]\n",
    "    #generate_mask -> [4, gen_lens-256]\n",
    "\n",
    "    #对数概率,求差就是求商,所以这里求的是新旧概率的变化率\n",
    "    #[4, gen_lens-256]\n",
    "    ratio = ((prob_new - prob_old) * generate_mask).exp()\n",
    "\n",
    "    #delta是估计出来的去基线Q值,以变化率来缩放Q值\n",
    "    #最大化Q值,以此来寻找最优的actor\n",
    "    #裁剪,防止自举\n",
    "    #[4, gen_lens-256]\n",
    "    loss1 = delta * ratio\n",
    "    loss2 = delta * ratio.clamp(0.8, 1.2)\n",
    "    loss = torch.min(loss1, loss2) * generate_mask\n",
    "    loss = loss.sum() / generate_mask.sum() / 8\n",
    "    return -loss\n",
    "\n",
    "\n",
    "loss_actor = get_loss_actor(prob_old, prob_old, delta, generate_mask)\n",
    "\n",
    "loss_actor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c9a4db24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.4388, device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_critic(value_new, value_old, delta, generate_mask):\n",
    "    value_new = value_new[:, 255:]\n",
    "    value_old = value_old[:, 255:]\n",
    "    generate_mask = generate_mask[:, 256:]\n",
    "\n",
    "    #value_new -> [4, gen_lens-256]\n",
    "    #value_old -> [4, gen_lens-256]\n",
    "    #delta -> [4, gen_lens-256]\n",
    "    #generate_mask -> [4, gen_lens-256]\n",
    "\n",
    "    #delta是估计出来的去基线Q值,加上value_old后还原为Q值\n",
    "    #value_new和Q值求mse loss即可,因为value都是对Q函数的估计\n",
    "    #裁剪,防止自举\n",
    "    #[4, gen_lens-256]\n",
    "    loss1 = (value_new - delta - value_old)**2\n",
    "    value_new = value_new.clamp(value_old - 0.2, value_old + 0.2)\n",
    "    loss2 = (value_new - delta - value_old)**2\n",
    "\n",
    "    #求平均\n",
    "    loss = torch.max(loss1, loss2) * generate_mask\n",
    "    loss = loss.sum() / 2 / generate_mask.sum() / 8\n",
    "\n",
    "    return loss\n",
    "\n",
    "\n",
    "loss_critic = get_loss_critic(value_old, value_old, delta, generate_mask)\n",
    "\n",
    "loss_critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "825b4816",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.05860239639878273, 0.14592543244361877)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def train(generate, generate_mask, prob_old, prob_ref, value_old, reward,\n",
    "          do_step):\n",
    "    #generate -> [4, gen_lens]\n",
    "    #generate_mask -> [4, gen_lens]\n",
    "    #prob_old -> [4, gen_lens-1]\n",
    "    #prob_ref -> [4, gen_lens-1]\n",
    "    #value_old -> [4, gen_lens-1]\n",
    "    #reward -> [4]\n",
    "    #do_step -> bool\n",
    "\n",
    "    #求出每句话结束的索引\n",
    "    #[4]\n",
    "    end = generate_mask[:, 256:].sum(1) + 255\n",
    "    end = end.tolist()\n",
    "\n",
    "    #结束以后的value归零\n",
    "    for i, e in enumerate(end):\n",
    "        value_old[i, e + 1:] = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        #计算新旧概率的kl散度,再把reward加在最后一个字上\n",
    "        #[4, gen_lens-1]\n",
    "        reward_kl = get_reward_kl(end, prob_old, prob_ref, reward)\n",
    "\n",
    "        #估计去基线的Q值\n",
    "        #[4, gen_lens-256]\n",
    "        delta = get_delta(value_old, reward_kl)\n",
    "\n",
    "    #重新计算回答被生成的概率\n",
    "    #[4, gen_lens-1]\n",
    "    prob_new = model_actor(input_ids=generate,\n",
    "                           attention_mask=generate_mask).logits\n",
    "    prob_new = get_prob(prob_new[:, :-1], generate[:, 1:])\n",
    "\n",
    "    #更新actor\n",
    "    loss_actor = get_loss_actor(prob_new, prob_old, delta, generate_mask)\n",
    "    accelerator.backward(loss_actor)\n",
    "    if do_step:\n",
    "        accelerator.clip_grad_norm_(\n",
    "            [i for i in model_actor.parameters() if i.requires_grad], 1.0)\n",
    "        optimizer_actor.step()\n",
    "        scheduler_actor.step()\n",
    "        optimizer_actor.zero_grad()\n",
    "\n",
    "    #重新计算每个词的value\n",
    "    #[4, gen_lens-1]\n",
    "    value_new = model_critic.get_value(input_ids=generate,\n",
    "                                       attention_mask=generate_mask)[:, :-1]\n",
    "\n",
    "    #更新critic\n",
    "    loss_critic = get_loss_critic(value_new, value_old, delta, generate_mask)\n",
    "    accelerator.backward(loss_critic)\n",
    "    if do_step:\n",
    "        accelerator.clip_grad_norm_(model_critic.parameters(), 1.0)\n",
    "        optimizer_critic.step()\n",
    "        scheduler_critic.step()\n",
    "        optimizer_critic.zero_grad()\n",
    "\n",
    "    return loss_actor.item(), loss_critic.item()\n",
    "\n",
    "\n",
    "train(generate, generate_mask, prob_old, prob_ref, value_old, reward, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4bd3e547",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 7144 0.06728362292051315 0.13812726736068726 2.0000000000000002e-07 1.0000000000000001e-07\n",
      "SELECT FATHERS_NAME from TABLE_NAME_89 where SPOUSER = \"OTTOKAR II\" and BIRTH = \"1204\"</s>\n",
      "-4.665192604064941\n",
      "15 7144 0.05079767480492592 0.13388875126838684 3.0000000000000004e-07 1.5000000000000002e-07\n",
      "SELECT division_title FROM table_name_87 WHERE playoffs_appearances = \"23\" AND conference_titles = \"1\"</s>\n",
      "-4.969192028045654\n",
      "23 7144 -0.02076289989054203 0.07107175141572952 4.0000000000000003e-07 2.0000000000000002e-07\n",
      "SELECT MAX(TOTAL) FROM table_name_61 WHERE gold < 1 AND bronze < 1 AND silver < 1 AND gold > 1 AND bronze > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND gold > 1 AND silver > 1 AND gold > 1 AND bronze > 1 AND silver > 1\n",
      "1.0344781875610352\n",
      "31 7144 0.021626608446240425 0.15791529417037964 5.000000000000001e-07 2.5000000000000004e-07\n",
      "SELECT opponent FROM table_name_73 WHERE record = \"86-62\"</s>\n",
      "-5.610930442810059\n",
      "39 7144 0.04333529621362686 0.12650182843208313 6.000000000000001e-07 3.0000000000000004e-07\n",
      "SELECT goals FROM table_name_6 WHERE country = \"france\"</s>\n",
      "-5.845401287078857\n",
      "47 7144 -0.0010221455013379455 0.1052459329366684 7.000000000000001e-07 3.5000000000000004e-07\n",
      "select MIN(pick__number) from TABLE_NAME_97 where POSITION = \"defensive tackle\" and PLAYER = \"Lorenzo Freeman\"</s>\n",
      "5.297737121582031\n",
      "55 7144 0.008534885011613369 0.11543802171945572 8.000000000000001e-07 4.0000000000000003e-07\n",
      "SELECT others FROM table_name_10 WHERE szdsz = \"4%\" AND fidesz = \"60%\"</s>\n",
      "-5.172423839569092\n",
      "63 7144 -0.02657955326139927 0.10900154709815979 9.000000000000001e-07 4.5000000000000003e-07\n",
      "select YEAR(s__won) from TABLE_NAME_42 where TOTAL < 293 and PLAYER = \"hale irwin\"</s>\n",
      "5.5256757736206055\n",
      "71 7144 -0.04583214968442917 0.10045618563890457 1.0000000000000002e-06 5.000000000000001e-07\n",
      "select TYPE_OF_GAME from TABLE_NAME_17 where RESULTS¹ = \"0:0\"</s>\n",
      "4.633450031280518\n",
      "79 7144 -0.046569887548685074 0.10667181015014648 1.1e-06 5.5e-07\n",
      "select RECORD from TABLE_NAME_38 where DATE = \"NOVEMBER 12\"</s>\n",
      "6.219855785369873\n",
      "87 7144 -0.01407866645604372 0.10831796377897263 1.2000000000000002e-06 6.000000000000001e-07\n",
      "select RESULT from TABLE_NAME_79 where DATE = \"NOVEMBER 24, 2002\"</s>\n",
      "5.642355442047119\n",
      "95 7144 -0.015037435106933117 0.10266099870204926 1.3e-06 6.5e-07\n",
      "select SURFACE from TABLE_NAME_16 where SCORE = \"4–6, 6–4, 3–6\"</s>\n",
      "5.736869812011719\n",
      "103 7144 -0.044633008539676666 0.1278841197490692 1.4000000000000001e-06 7.000000000000001e-07\n",
      "select YEAR from TABLE_NAME_58 where CATALOGUE = \"รอเธอหันมา – โฟกัส\"</s>\n",
      "4.945871829986572\n",
      "111 7144 -0.03414836898446083 0.09234659373760223 1.5e-06 7.5e-07\n",
      "select SOUND_PRICE from TABLE_2638104_1 where CALL_SIGNAL_PRICE = \"JOAB-DTV\"</s>\n",
      "5.50108003616333\n",
      "119 7144 0.004100877325981855 0.0924663320183754 1.6000000000000001e-06 8.000000000000001e-07\n",
      "select RUNNER from TABLE_NAME_80 where TOTAL = \"2\"</s>\n",
      "5.719141483306885\n",
      "127 7144 -0.010415674187242985 0.10225076973438263 1.7000000000000002e-06 8.500000000000001e-07\n",
      "select 2006 FROM table_2328113_1 WHERE mother_tongue = \"poles\"</s>\n",
      "5.148213863372803\n",
      "135 7144 -0.01642061024904251 0.09112893044948578 1.8000000000000001e-06 9.000000000000001e-07\n",
      "select TABLE_NAME_83 from TABLE_NAME_82 where RANK > 7</s>\n",
      "5.647308826446533\n",
      "143 7144 0.011298622004687786 0.07250917702913284 1.9000000000000002e-06 9.500000000000001e-07\n",
      "select max(speed__ghz_) from TABLE_269920_16 where MODEL = \"W3540\"</s>\n",
      "5.065927505493164\n",
      "151 7144 -0.047582484781742096 0.05529959127306938 2.0000000000000003e-06 1.0000000000000002e-06\n",
      "select HOME_TEAM from TABLE_NAME_64 where HOME_TEAM = \"NORTH MELBOURNE\"</s>\n",
      "5.073512077331543\n",
      "159 7144 -0.012967714108526707 0.043142594397068024 2.1000000000000002e-06 1.0500000000000001e-06\n",
      "select settlement FROM table_2562572_26 WHERE cyrillic_name_other_names = \"đurđin\"</s>\n",
      "4.340605735778809\n",
      "167 7144 0.0012918668799102306 0.042974136769771576 2.2e-06 1.1e-06\n",
      "select TIME from TABLE_NAME_86 where WRESTLER = \"NATALYA\"</s>\n",
      "5.190241813659668\n",
      "175 7144 0.02245437167584896 0.034979965537786484 2.3000000000000004e-06 1.1500000000000002e-06\n",
      "select max(upstream) from TABLE_17304621_14 where Price_TL = \"39 tl\"</s>\n",
      "5.136510848999023\n",
      "183 7144 0.02336741052567959 0.03354145213961601 2.4000000000000003e-06 1.2000000000000002e-06\n",
      "select STATUS from TABLE_19061741_1 where NAME = \"nel rapiz\"</s>\n",
      "5.090132236480713\n",
      "191 7144 0.0033920013811439276 0.024983355775475502 2.5e-06 1.25e-06\n",
      "select LAND_sqmi__ from TABLE_NAME_79 where WATER_sqmi__ < 0.04 and LONGITUDE_96.79477062</s>\n",
      "5.003951549530029\n",
      "199 7144 -0.02530285343527794 0.01723288558423519 2.6e-06 1.3e-06\n",
      "select max(YEAR) from TABLE_NAME_52 where LAPS > 196</s>\n",
      "5.4529266357421875\n",
      "207 7144 -0.013800928369164467 0.0209150779992342 2.7000000000000004e-06 1.3500000000000002e-06\n",
      "select count(*) from TABLE_22171978_1 where LOCATION = \"naga, camarines sur\"</s>\n",
      "4.774313449859619\n",
      "215 7144 -0.001177821308374405 0.027951452881097794 2.8000000000000003e-06 1.4000000000000001e-06\n",
      "select TIME from TABLE_NAME_78 where OPPONENT = \"Satoshi Honma\"</s>\n",
      "5.983563423156738\n",
      "223 7144 -0.024631276726722717 0.02361643873155117 2.9e-06 1.45e-06\n",
      "select max(YEAR_WON) from TABLE_NAME_6 where FINISH = \"t24\" and COUNTRY = \"england\"</s>\n",
      "4.975122451782227\n",
      "231 7144 0.006516146007925272 0.021238837391138077 3e-06 1.5e-06\n",
      "select TEAM_NAME from TABLE_25421463_1 where POINTS < 38</s>\n",
      "5.242241859436035\n",
      "239 7144 0.015232661738991737 0.02996467798948288 3.1000000000000004e-06 1.5500000000000002e-06\n",
      "select max(Id) from TABLE_NAME_82 where GARION_WELLER_POSITION = \"1\"</s>\n",
      "5.330432891845703\n",
      "247 7144 0.017916494980454445 0.023382633924484253 3.2000000000000003e-06 1.6000000000000001e-06\n",
      "select entrant from TABLE_NAME_68 where ENGINE = \"COSWORTH DFV V8\" and CHASSIS = \"DN9\"</s>\n",
      "4.971756935119629\n",
      "255 7144 0.02550511807203293 0.019358808174729347 3.3000000000000006e-06 1.6500000000000003e-06\n",
      "select count(*) from artist_id group by name order by artist_id desc limit 1</s>\n",
      "5.09409236907959\n",
      "263 7144 0.03429676592350006 0.027026919648051262 3.4000000000000005e-06 1.7000000000000002e-06\n",
      "select count(a) from TABLE_11452830_2 where KICKOFF_ = \"SEPTEMBER 4, 1988\" and DATE = \"SEPTEMBER 4, 1988\"</s>\n",
      "5.575100421905518\n",
      "271 7144 0.012491417117416859 0.0182326752692461 3.5e-06 1.75e-06\n",
      "select COUNT(*) from PRODUCTS where PRODUCT_CODE = \"t\" and DESCRIPTION = \"t\"</s>\n",
      "5.350679874420166\n",
      "279 7144 -0.024982016533613205 0.018977386876940727 3.6000000000000003e-06 1.8000000000000001e-06\n",
      "select DIFFICULTY from TABLE_24463470_1 where CONSTRUCTION_CIRCUIT = \"AThens\"</s>\n",
      "5.973182678222656\n",
      "287 7144 -0.03571067377924919 0.016895566135644913 3.7e-06 1.85e-06\n",
      "select MIN(YEAR) from TABLE_NAME_75 where COUNTRY = \"UNITED STATES\" and ICONS = \"TIGER WOODS\"</s>\n",
      "6.118460178375244\n",
      "295 7144 -0.028237374499440193 0.01857234723865986 3.8000000000000005e-06 1.9000000000000002e-06\n",
      "select COUNT(LEADER_AT_THE_SUMMIT) from TABLE_NAME_82 where YEAR = \"1956\" and FINISH = \"Carcassonne\"</s>\n",
      "6.076149940490723\n",
      "303 7144 -0.04463948309421539 0.018800051882863045 3.900000000000001e-06 1.9500000000000004e-06\n",
      "select RECORD from TABLE_14863869_1 where DATE = \"DECEMBER 2, 1984\"</s>\n",
      "5.858147144317627\n",
      "311 7144 -0.00535298977047205 0.014125790446996689 4.000000000000001e-06 2.0000000000000003e-06\n",
      "select nominated_work from TABLE_NAME_94 where award = \"tony award\" and year = \"2009\"</s>\n",
      "4.813504695892334\n",
      "319 7144 0.01201576553285122 0.01424696296453476 4.1e-06 2.05e-06\n",
      "select H/A/N from TABLE_NAME_50 where DATE = \"october 31\"</s>\n",
      "5.075416564941406\n",
      "327 7144 0.03139110654592514 0.017132893204689026 4.2000000000000004e-06 2.1000000000000002e-06\n",
      "select TOURNAMENT from TABLE_NAME_56 where OPEN_ID = \"2010\"</s>\n",
      "5.7537102699279785\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "335 7144 0.021731732413172722 0.019196977838873863 4.3e-06 2.15e-06\n",
      "select BIRTH_DATE from TABLE_NAME_3 where WEIGHT__KG_ = \"100\" and WORLD_RANK = \"2\"</s>\n",
      "5.755478858947754\n",
      "343 7144 0.010347575880587101 0.016551362350583076 4.4e-06 2.2e-06\n",
      "select 2009 from TABLE_NAME_71 where 2011 = \"SF\"</s>\n",
      "4.730398178100586\n",
      "351 7144 -0.006458021234720945 0.006772099528461695 4.5e-06 2.25e-06\n",
      "select DECILE from TABLE_NAME_99 where AUTHORITY = \"STATE AUTHORITY\" and ROLL = \"120\"</s>\n",
      "5.752123832702637\n",
      "359 7144 -0.018256478011608124 0.012758556753396988 4.600000000000001e-06 2.3000000000000004e-06\n",
      "select DATE from TABLE_NAME_49 where GAME > 69 and OPPONENT = \"ST. LOUIS BLUES\"</s>\n",
      "5.8697614669799805\n",
      "367 7144 -0.0244454313069582 0.007603432983160019 4.7e-06 2.35e-06\n",
      "select TEAM from TABLE_23285805_5 where DATE = \"DECEMBER 19\"</s>\n",
      "4.554546356201172\n",
      "375 7144 0.032890770584344864 0.028268737718462944 4.800000000000001e-06 2.4000000000000003e-06\n",
      "select MIN(silver) from TABLE_NAME_68 where GOLD < 0</s>\n",
      "5.411764144897461\n",
      "383 7144 0.0065108779817819595 0.009223856031894684 4.9000000000000005e-06 2.4500000000000003e-06\n",
      "select TYPE from SCHOOL_BUS where SCHOOL_ID = \"1\"</s>\n",
      "5.606931686401367\n",
      "391 7144 0.0005688492092303932 0.007761905901134014 5e-06 2.5e-06\n",
      "select REACT from TABLE_NAME_53 where LANE > 5 and TIME < 20.05</s>\n",
      "5.8783793449401855\n",
      "399 7144 -0.007981151342391968 0.00969310849905014 5.1e-06 2.55e-06\n",
      "select MIN(LOWEST_POINT) from TABLE_1416612_1 where HIGHEST_POINT = \"mount greylock\"</s>\n",
      "3.777118682861328\n",
      "407 7144 -0.02049568109214306 0.006928172428160906 5.2e-06 2.6e-06\n",
      "select HOME_TEAM from TABLE_NAME_20 where AWAY_TEAM = \"RICHMOND\"</s>\n",
      "5.424729824066162\n",
      "415 7144 0.0010277876863256097 0.005149020813405514 5.300000000000001e-06 2.6500000000000005e-06\n",
      "select RESULT from TABLE_NAME_84 where WEEK__NUMBER = \"TOP 11\"</s>\n",
      "5.447086334228516\n",
      "423 7144 0.002900225343182683 0.005808021407574415 5.400000000000001e-06 2.7000000000000004e-06\n",
      "select where_built from TABLE_NAME_79 where YEAR_BUILED = \"1858\"</s>\n",
      "5.286684989929199\n",
      "431 7144 0.0056602368131279945 0.004740270785987377 5.500000000000001e-06 2.7500000000000004e-06\n",
      "select MIN(POINTS__NON_PODIUM) from TABLE_2822193_1 where SERIES = \"V8 SUPERCAR\"</s>\n",
      "6.0959625244140625\n",
      "439 7144 0.009970337152481079 0.00584759283810854 5.600000000000001e-06 2.8000000000000003e-06\n",
      "select MIN(bonus_points) from TABLE_NAME_47 where POINTS_AGAINST =448</s>\n",
      "5.867838382720947\n",
      "447 7144 -0.011238002218306065 0.010818610899150372 5.7e-06 2.85e-06\n",
      "select MENS_DOUBLES from TABLE_15002265_1 where MENS_SINGLES = \"jürgen koch\"</s>\n",
      "3.7094173431396484\n",
      "455 7144 -0.03156742453575134 0.009099275805056095 5.8e-06 2.9e-06\n",
      "select WEEK from TABLE_NAME_20 where DATE = \"OCTOBER 21, 1974\" and ATTENDANCE = \"LESS THAN 50,623\"</s>\n",
      "5.901475429534912\n",
      "463 7144 -0.018218830227851868 0.004264476243406534 5.9e-06 2.95e-06\n",
      "select OPPONENTS from TABLE_NAME_80 where DATE = \"NOVEMBER 25\"</s>\n",
      "5.621314525604248\n",
      "471 7144 0.004775394685566425 0.004266096744686365 6e-06 3e-06\n",
      "select COLLEGE_JUNIOR_CLUB_TEAM from TABLE_1965650_7 where NHL_TEAM = \"BUFFALO SABRES\"</s>\n",
      "5.690877914428711\n",
      "479 7144 0.008845270611345768 0.004344435874372721 6.1e-06 3.05e-06\n",
      "select TITLE from TABLE_27397948_2 where NO_IN_SERIES = \"57\"</s>\n",
      "5.660006046295166\n",
      "487 7144 -0.01009136438369751 0.0066725690849125385 6.200000000000001e-06 3.1000000000000004e-06\n",
      "select TRANSMITTER_LOCATION from TABLE_NAME_87 where STATION = \"VOICE_OF_VIRTANAM\"</s>\n",
      "4.871917247772217\n",
      "495 7144 0.008440857753157616 0.005403838586062193 6.300000000000001e-06 3.1500000000000003e-06\n",
      "select SCORE_IN_THE_FINAL from TABLE_NAME_90 where PARTNER = \"LINNIE LAWRENCE LAWRENCE\"</s>\n",
      "6.172532558441162\n",
      "503 7144 -0.005126327276229858 0.003012068336829543 6.4000000000000006e-06 3.2000000000000003e-06\n",
      "select LENGTH_FUEL from TABLE_NAME_22 where QUANTITY = 30</s>\n",
      "5.445951461791992\n",
      "511 7144 0.002220569644123316 0.003923568408936262 6.5000000000000004e-06 3.2500000000000002e-06\n",
      "select AWARD from TABLE_NAME_85 where TITLE = \"exorcist\" and CYBER_CODE = \"98\" and CYBER_CODE_2 = \"sounds\"</s>\n",
      "5.544036865234375\n",
      "519 7144 0.0014248555526137352 0.0037991872522979975 6.600000000000001e-06 3.3000000000000006e-06\n",
      "select tourism receipts 2003 from TABLE_18524_6 where COUNTRY = \"COLOMBIA\"</s>\n",
      "4.535700798034668\n",
      "527 7144 -0.004141851328313351 0.004762345924973488 6.700000000000001e-06 3.3500000000000005e-06\n",
      "select MIN(pick__number) from TABLE_15353123_1 where POSITION = \"defensive end\"</s>\n",
      "4.4780426025390625\n",
      "535 7144 0.00838775560259819 0.004114083014428616 6.800000000000001e-06 3.4000000000000005e-06\n",
      "select PARTY from TABLE_19753079_35 where DISTRICT = \"new york 4\"</s>\n",
      "4.705925941467285\n",
      "543 7144 -0.0006580307963304222 0.0019911157432943583 6.9e-06 3.45e-06\n",
      "select GAME from TABLE_NAME_87 where LOCATION = \"TOKYO\" and POINTS = \"31\"</s>\n",
      "6.268172264099121\n",
      "551 7144 -0.001188851660117507 0.0034132408909499645 7e-06 3.5e-06\n",
      "select WINS from TABLE_NAME_29 where RANK > 42 and COLLEGE > WASHINGTON_STATE_CITY_COUNTY_ and LOSSES > 980 and FIRST_SEASON > 1906</s>\n",
      "5.828269004821777\n",
      "559 7144 -0.006912268232554197 0.004142226185649633 7.100000000000001e-06 3.5500000000000003e-06\n",
      "select CITY from TABLE_NAME_78 where RESULTS¹ = \"0:0\"</s>\n",
      "4.260807514190674\n",
      "567 7144 0.010292638093233109 0.00413592578843236 7.2000000000000005e-06 3.6000000000000003e-06\n",
      "select CAPACITY_IN_PERSONS_HOUR from TABLE_17814458_1 where CONSTRUCTION_YEAR_S_ = \" 1983\"</s>\n",
      "5.011470794677734\n",
      "575 7144 0.0034137656912207603 0.001935747917741537 7.3e-06 3.65e-06\n",
      "select DRAWS from TABLE_NAME_25 where PLAYED = 38</s>\n",
      "5.949048042297363\n",
      "583 7144 -0.01460003387182951 0.0028098071925342083 7.4e-06 3.7e-06\n",
      "select DATE from TABLE_17355408_7 where HIGH_ASSISTS = \"6\"</s>\n",
      "5.23064661026001\n",
      "591 7144 -0.005173452664166689 0.002818835200741887 7.500000000000001e-06 3.7500000000000005e-06\n",
      "select Jockey from TABLE_NAME_50 where Horse_NAME = \"Hard Spun\"</s>\n",
      "4.829578399658203\n",
      "599 7144 0.0075003947131335735 0.0022612405009567738 7.600000000000001e-06 3.8000000000000005e-06\n",
      "select WRITTEN_BY from TABLE_15284274_1 where DIRECTED_BY = \"WILL WARNING\" and NO_IN_SEASON = \"EPISODE 11\"</s>\n",
      "4.879549026489258\n",
      "607 7144 0.015713956207036972 0.0034660452511161566 7.7e-06 3.85e-06\n",
      "select COUNT( POLISH_CUP) from TABLE_NAME_34 where PLAYER = \"MACIEJ IWAński\" and EKSTRAKLAASA < 1</s>\n",
      "5.323832988739014\n",
      "615 7144 -0.004256528336554766 0.0023018307983875275 7.800000000000002e-06 3.900000000000001e-06\n",
      "select POSITION from TABLE_NAME_10 where OVERALL = 106 and COLLEGE = \"College of Virginia\"</s>\n",
      "5.354609966278076\n",
      "623 7144 -0.010512792505323887 0.0035152544733136892 7.9e-06 3.95e-06\n",
      "select SEASON from TABLE_2140071_7 where ORIGINAL_SEASON = \"2012\"</s>\n",
      "5.241788864135742\n",
      "631 7144 0.010744833387434483 0.0019611194729804993 8.000000000000001e-06 4.000000000000001e-06\n",
      "select FINALS from TABLE_NAME_71 where PRELIM = \"out of playoffs\"</s>\n",
      "4.6492156982421875\n",
      "639 7144 0.01270052045583725 0.002797413384541869 8.1e-06 4.05e-06\n",
      "select HOME_TEAM from TABLE_NAME_28 where AWAY_TEAM = \"collingwood\"</s>\n",
      "5.487864971160889\n",
      "647 7144 -0.014891624450683594 0.0027995165437459946 8.2e-06 4.1e-06\n",
      "select 2007 FROM table_23619212_1 WHERE 2008 = \"1.7%\"</s>\n",
      "5.00126838684082\n",
      "655 7144 -0.01875333860516548 0.00307798245921731 8.3e-06 4.15e-06\n",
      "select count(no) from TABLE_2866509_1 where US_VIEWERS__MILLIONS_ = \"5.60\"</s>\n",
      "5.499658584594727\n",
      "663 7144 -0.0007447642274200916 0.001479871105402708 8.400000000000001e-06 4.2000000000000004e-06\n",
      "select ELECTION from TABLE_NAME_11 where ELECTION = \"1885\"</s>\n",
      "5.280425548553467\n",
      "671 7144 0.019417298957705498 0.003533429466187954 8.5e-06 4.25e-06\n",
      "select TOPAR from TABLE_NAME_37 where TOTAL = 297 and FINISH = \"1\"</s>\n",
      "5.212052345275879\n",
      "679 7144 0.021829675883054733 0.0039006678853183985 8.6e-06 4.3e-06\n",
      "select SERIE_A from TABLE_NAME_79 where COPPAPA_ITALIA = \"5\"</s>\n",
      "6.014427185058594\n",
      "687 7144 -0.007296100724488497 0.0012915239203721285 8.700000000000001e-06 4.350000000000001e-06\n",
      "select GAME from TABLE_NAME_47 where TIME < 27,374 and ATTENDANCE < 27,374</s>\n",
      "5.772522449493408\n",
      "695 7144 -0.0004460676573216915 0.0017173175001516938 8.8e-06 4.4e-06\n",
      "select TEAM from TABLE_NAME_83 where KIT_MARKET_UMBRO = \"UNBRO\" and CAPTAIN = \"GARY MABBUT\"</s>\n",
      "5.586217403411865\n",
      "703 7144 -0.0008659572922624648 0.0008995558600872755 8.900000000000001e-06 4.450000000000001e-06\n",
      "select EPISODE from TABLE_NAME_62 where SHARE > 11.26 and WEEKLY_RANK > 8</s>\n",
      "5.3747358322143555\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "711 7144 0.006517982110381126 0.0014100588159635663 9e-06 4.5e-06\n",
      "select MIN(GUEST) from TABLE_25691838_12 where PRODUCTION_CODE = \"6152\"</s>\n",
      "5.6593546867370605\n",
      "719 7144 -0.008777322247624397 0.0019637083169072866 9.100000000000001e-06 4.5500000000000005e-06\n",
      "select MUSIC from TABLE_NAME_60 where RECORD_LABEL = \"MOTIVATE BOX\" and RECORD_WAVES = \"MOTIVATE RECORD\"</s>\n",
      "5.037847518920898\n",
      "727 7144 -0.0027400683611631393 0.0008685711654834449 9.200000000000002e-06 4.600000000000001e-06\n",
      "select DOWN__UP_TO_KBIT_S from TABLE_1773908_3 where PROVIDER = \"DEUTSCHE TELEKOM AG\"</s>\n",
      "5.676542282104492\n",
      "735 7144 0.015366323292255402 0.0020056741777807474 9.3e-06 4.65e-06\n",
      "select AUTHOR from TABLE_NAME_16 where UNIT = \"MALASUDA\" and NAME = \"VALENTINA\"</s>\n",
      "5.780348300933838\n",
      "743 7144 0.004614296369254589 0.0006417194381356239 9.4e-06 4.7e-06\n",
      "select METHOD from TABLE_NAME_19 where OPPONENT = \"Thiago Alves\"</s>\n",
      "4.922232627868652\n",
      "751 7144 -0.01691412553191185 0.002768877660855651 9.5e-06 4.75e-06\n",
      "select SOCKET from TABLE_NAME_30 where FREQUENCY = \"2200MHZ\" and RELEASE_DATE = \"june 4, 2008\"</s>\n",
      "5.385894775390625\n",
      "759 7144 -0.012874254025518894 0.002233229111880064 9.600000000000001e-06 4.800000000000001e-06\n",
      "select MENS_DOUBLES from TABLE_14903627_1 where WOMENS_DOUBLES = \"PIRET HAMER HELEN REINO\"</s>\n",
      "4.786398410797119\n",
      "767 7144 0.010203363373875618 0.001337365130893886 9.7e-06 4.85e-06\n",
      "select avg(REGULATORY_CITATION) from TABLE_NAME_53 where AVERAGING_TIME = \"40 cfr 50.4(b)\"</s>\n",
      "5.568762302398682\n",
      "775 7144 0.007148880045861006 0.0014411440351977944 9.800000000000001e-06 4.9000000000000005e-06\n",
      "select YEAR from TABLE_NAME_92 where FINISH = \"19\"</s>\n",
      "5.361931800842285\n",
      "783 7144 -0.02411278523504734 0.00408896803855896 9.9e-06 4.95e-06\n",
      "select TIME from TABLE_NAME_83 where METHOD = \"TKO\"</s>\n",
      "5.332720756530762\n",
      "791 7144 -0.010287612676620483 0.0018185412045568228 1e-05 5e-06\n",
      "select avg(CAPS) from TABLE_NAME_6 where SCOTLAND_CAREER = \"1986–1998\"</s>\n",
      "5.30741548538208\n",
      "799 7144 0.0073458049446344376 0.001082172035239637 9.999949644960027e-06 4.9999748224800135e-06\n",
      "select WEEKS_ON_TOP from TABLE_NAME_94 where ISSUE_DATE_S_ = \"20 NOVEMBER\"</s>\n",
      "5.3102850914001465\n",
      "807 7144 0.008317342028021812 0.0011343149235472083 9.999798580854356e-06 4.999899290427178e-06\n",
      "select YEAR from TABLE_NAME_65 where POINTS > 0</s>\n",
      "5.706164360046387\n",
      "815 7144 -0.015395025722682476 0.00198234710842371 9.999546810725726e-06 4.999773405362863e-06\n",
      "select TOURNAMENT from TABLE_NAME_33 where OPEN = \"1r\"</s>\n",
      "5.267430305480957\n",
      "823 7144 -0.016464581713080406 0.003436700440943241 9.999194339645292e-06 4.999597169822646e-06\n",
      "select T1.transaction_type_code from T1.transactions group by T1.transaction_type_description</s>\n",
      "5.674699306488037\n",
      "831 7144 0.014072327874600887 0.0016666438896209002 9.998741174712534e-06 4.999370587356267e-06\n",
      "select WON from TABLE_2509505_1 where GOALS_FOR = \"274\"</s>\n",
      "4.547619342803955\n",
      "839 7144 0.00566536420956254 0.001390613615512848 9.998187325055107e-06 4.9990936625275535e-06\n",
      "select KIND_OF_THE_SONG from TABLE_NAME_66 where SINGER = \"MIRIAM YEUNG\"</s>\n",
      "5.132743835449219\n",
      "847 7144 -0.015152224339544773 0.0016742913285270333 9.997532801828659e-06 4.998766400914329e-06\n",
      "select HEAD_OF_HOUSEHOLDS from TABLE_NAME_39 where TASK_NO = \"7\"</s>\n",
      "5.71763277053833\n",
      "855 7144 -0.022289933636784554 0.002868392737582326 9.996777618216608e-06 4.998388809108304e-06\n",
      "select RESULT from TABLE_NAME_34 where CITY = \"MILWAUKEE\"</s>\n",
      "5.995124816894531\n",
      "863 7144 0.015967989340424538 0.003355406690388918 9.995921789429875e-06 4.997960894714937e-06\n",
      "select TEAM from TABLE_27902171_8 where RECORD = \"36-29\"</s>\n",
      "4.766633033752441\n",
      "871 7144 0.009847188368439674 0.0015686090337112546 9.994965332706574e-06 4.997482666353287e-06\n",
      "select PLAYER from TABLE_NAME_27 where YEARS_AT_CLUB = \"1943\" and GOALS = \"2\" and DEBUT_YEAR = \"1942\"</s>\n",
      "6.009763240814209\n",
      "879 7144 -0.00918938871473074 0.0010553847532719374 9.993908267311677e-06 4.996954133655838e-06\n",
      "select LAPS from TABLE_NAME_92 where MANUFACTURER = \"suzuki\" and TIME_RETIRED = \"+1:02.804\"</s>\n",
      "5.332784652709961\n",
      "887 7144 -0.0007300430443137884 0.0022202262189239264 9.992750614536606e-06 4.996375307268303e-06\n",
      "select REPORT from TABLE_NAME_25 where CONSTRUCTION_CIRCUIT = \"MODENA\"</s>\n",
      "4.222877502441406\n",
      "895 7144 -0.008320845663547516 0.0010117695201188326 9.991492397698825e-06 4.995746198849413e-06\n",
      "select YEAR from TABLE_NAME_87 where COUNT(FIREDAMP) > 11</s>\n",
      "5.3542962074279785\n",
      "903 7144 0.030831772834062576 0.007358897011727095 9.990133642141359e-06 4.9950668210706795e-06\n",
      "select COUNT(district) from DISTRICT_CITY_APPEARANCE where CITY_APPEARANCE > \"average city area\"</s>\n",
      "5.891624450683594\n",
      "911 7144 0.014378669671714306 0.0020629139617085457 9.98867437523228e-06 4.99433718761614e-06\n",
      "select POINTS from TABLE_NAME_32 where CHASSIS = \"BRM p160e\" and YEAR > 1974</s>\n",
      "5.096571922302246\n",
      "919 7144 -0.0026948435697704554 0.000974501483142376 9.987114626364172e-06 4.993557313182086e-06\n",
      "select SCORE from TABLE_17311759_4 where TEAM = \"Charlotte\"</s>\n",
      "5.314047813415527\n",
      "927 7144 0.0007583866827189922 0.0009124202770181 9.985454426953513e-06 4.992727213476757e-06\n",
      "select SURFACE from TABLE_16741821_8 where OPPONENT = \"GEORGE KHRIKADZE\"</s>\n",
      "4.864712715148926\n",
      "935 7144 -0.012916159816086292 0.001216038246639073 9.983693810440073e-06 4.991846905220037e-06\n",
      "select RESULT from TABLE_NAME_17 where YEAR > 2007</s>\n",
      "5.770564079284668\n",
      "943 7144 -0.00573897548019886 0.000739460694603622 9.981832812286217e-06 4.990916406143108e-06\n",
      "select ORIGINAL_AIR_DATE from TABLE_19266557_1 where NUMBER = \"7\"</s>\n",
      "5.35303258895874\n",
      "951 7144 0.022879112511873245 0.0033744981046766043 9.979871469976197e-06 4.989935734988098e-06\n",
      "select OVERALL from TABLE_NAME_71 where PLYFF = \"0-1\" and AWAY = \"1-1\"</s>\n",
      "5.634341239929199\n",
      "959 7144 0.030846916139125824 0.005944300442934036 9.9778098230154e-06 4.9889049115077e-06\n",
      "select REGION from TABLE_14532_1 where CAPITAL = \"MILAN\"</s>\n",
      "5.452981472015381\n",
      "967 7144 0.01623508334159851 0.0022991385776549578 9.975647912929558e-06 4.987823956464779e-06\n",
      "select MINISTER from PARTY where LEFT_OFFICE = \"LEEANOR\"</s>\n",
      "6.529881477355957\n",
      "975 7144 -0.006287744268774986 0.0015471584629267454 9.973385783263893e-06 4.9866928916319465e-06\n",
      "select STADIUM from TABLE_NAME_35 where FINAL_SCORE = \"20-10\"</s>\n",
      "4.908454895019531\n",
      "983 7144 -0.018680177628993988 0.0023032876197248697 9.971023479582258e-06 4.985511739791129e-06\n",
      "select ORIGINAL_USA_AIR_DATE from TABLE_NAME_80 where ORIGINAL_CANADA_AIR_DATE = \"DECEMBER 9, 2007\"</s>\n",
      "5.081233024597168\n",
      "991 7144 -0.015239577740430832 0.0018523231847211719 9.968561049466214e-06 4.984280524733107e-06\n",
      "select Format FROM table_name_81 WHERE Label = \"alfa records Label\" AND Catalog = \"alca-9016\"</s>\n",
      "4.937170505523682\n",
      "999 7144 0.018500836566090584 0.003326444188132882 9.965998542514066e-06 4.982999271257033e-06\n",
      "select max(EARNING___) from TABLE_NAME_60 where PLAYER = \"CURTIS SIMMONS\"</s>\n",
      "5.8982977867126465\n",
      "1007 7144 0.020770274102687836 0.0025690263137221336 9.963336010339869e-06 4.981668005169934e-06\n",
      "select COVERAGE from TABLE_2523809_1 where CALLSIGN = \"KPNZ-TV\"</s>\n",
      "5.504032135009766\n",
      "1015 7144 0.006965084001421928 0.0005517688114196062 9.960573506572391e-06 4.980286753286196e-06\n",
      "select DATE from TABLE_NAME_75 where OPPONENT = \"SEATTLE MARINERS\"</s>\n",
      "5.6882195472717285\n",
      "1023 7144 -0.01786186173558235 0.0023885932750999928 9.957711086854025e-06 4.978855543427012e-06\n",
      "select YEAR_NAMED from TABLE_16799784_4 where NAME = \"OTAFUKUKU THOLI\"</s>\n",
      "5.508418560028076\n",
      "1031 7144 -0.030710522085428238 0.004783193580806255 9.954748808839675e-06 4.977374404419838e-06\n",
      "select HOME_TEAM from TABLE_NAME_78 where VENUE = \"KARDINIA PARK\"</s>\n",
      "5.590004920959473\n",
      "1039 7144 -0.020925305783748627 0.0023890978191047907 9.951686732195593e-06 4.9758433660977966e-06\n",
      "select LOCATION_ATTENDANCE from TABLE_27902171_7 where HIGH_REBOUNDS = \"10\"</s>\n",
      "5.983992576599121\n",
      "1047 7144 0.013212288729846478 0.001413134508766234 9.948524918598175e-06 4.974262459299088e-06\n",
      "select count(FIRST_ELECTED) from TABLE_1342233_32 where INCUMBENT = \"sOL BLOOM\"</s>\n",
      "4.508626937866211\n",
      "1055 7144 0.011841352097690105 0.0012937007704749703 9.945263431732722e-06 4.972631715866361e-06\n",
      "select DISTRICT from TABLE_26131768_4 where RESIDENCE = \"DUBLIN\"</s>\n",
      "5.148443222045898\n",
      "1063 7144 -7.490919961128384e-05 0.001496685785241425 9.941902337292156e-06 4.970951168646078e-06\n",
      "select MIN(main_industry) from company join main_industry_1 as main_industry_1 on MIN(main_industry_1) > MIN(main_industry_1)</s>\n",
      "5.513563632965088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1071 7144 -0.011808217503130436 0.0010893353028222919 9.938441702975689e-06 4.9692208514878445e-06\n",
      "select GOALS from TABLE_NAME_84 where ASSISTS = \"33\" and GAMES = \"85\"</s>\n",
      "5.852644443511963\n",
      "1079 7144 -0.005128490272909403 0.0010268338955938816 9.934881598487478e-06 4.967440799243739e-06\n",
      "select HOME_GAMES from TABLE_28884858_1 where AVERAGE_ATTENDANCE = \"79475\"</s>\n",
      "5.641493797302246\n",
      "1087 7144 0.01987536996603012 0.0021973333787173033 9.931222095535204e-06 4.965611047767602e-06\n",
      "select MIN(TOTAL) from TABLE_NAME_78 where NATION = \"canada (can)\" and BRONZE < 0</s>\n",
      "6.014262676239014\n",
      "1095 7144 0.01893678866326809 0.0025302923750132322 9.927463267828635e-06 4.963731633914317e-06\n",
      "select ENGLISH from TABLE_26614365_1 where EAST as \"light\"</s>\n",
      "5.618564605712891\n",
      "1103 7144 0.009567228145897388 0.0009888582862913609 9.923605191078134e-06 4.961802595539067e-06\n",
      "select CITY from TABLE_NAME_41 where ERP__ANALOG_DIGITAL = \"600kw 500kw\"</s>\n",
      "5.127023696899414\n",
      "1111 7144 -0.020534832030534744 0.0022058403119444847 9.91964794299315e-06 4.959823971496575e-06\n",
      "select VISITOR from TABLE_NAME_77 where DATE = \"12 MARCH 2008\"</s>\n",
      "5.595868110656738\n",
      "1119 7144 -0.03638080880045891 0.006210785824805498 9.915591603280632e-06 4.957795801640316e-06\n",
      "select ROLE_CODE from EMPLOYEES where largest_number_of_employments > 1</s>\n",
      "6.320826053619385\n",
      "1127 7144 -0.019986333325505257 0.0025615408085286617 9.911436253643445e-06 4.9557181268217225e-06\n",
      "select TEAM from TABLE_NAME_54 where INN = \"9th\"</s>\n",
      "5.248062610626221\n",
      "1135 7144 -0.001849850290454924 0.0005164009635336697 9.907181977778702e-06 4.953590988889351e-06\n",
      "select avg(TOTAL) from TABLE_NAME_38 where RANK_BY_AVERAGE < 12 and NUMBER_OF_DANCES < 3</s>\n",
      "5.750154495239258\n",
      "1143 7144 0.024503305554389954 0.0034015418495982885 9.902828861376101e-06 4.9514144306880506e-06\n",
      "select min(fa_cup_apps) from TABLE_27086613_5 where fa_cup_apps = \"2\" and position = \"2\"</s>\n",
      "4.719058990478516\n",
      "1151 7144 0.03088901937007904 0.006359890103340149 9.898376992116179e-06 4.949188496058089e-06\n",
      "select WOMENS_SINGLES from TABLE_12027364_1 where MENS_SINGLES = \"PETER rasmussen\"</s>\n",
      "5.24525260925293\n",
      "1159 7144 0.018935421481728554 0.0020033298060297966 9.89382645966856e-06 4.94691322983428e-06\n",
      "select OVERALL from TABLE_NAME_30 where NAME = \"BOB ANDERSON\"</s>\n",
      "5.758285999298096\n",
      "1167 7144 -0.005795566365122795 0.0007392637780867517 9.889177355690136e-06 4.944588677845068e-06\n",
      "select CANDIDATES from TABLE_1341973_3 where DISTRICT = \"ALA 6\"</s>\n",
      "5.847850322723389\n",
      "1175 7144 -0.004303136374801397 0.0007679451955482364 9.884429773823238e-06 4.942214886911619e-06\n",
      "select CLUBS_INTRODUCED from TABLE_NAME_59 where CLUBS_REMAINING > 4 and WINNERS_FROM_PREVIOUS_ROUND > 4</s>\n",
      "6.567437648773193\n",
      "1183 7144 -0.00964665412902832 0.0013622369151562452 9.879583809693737e-06 4.939791904846869e-06\n",
      "select TO_PAR from TABLE_NAME_42 where SCORE > 68</s>\n",
      "5.5022993087768555\n",
      "1191 7144 0.022839585319161415 0.004131836351007223 9.874639560909118e-06 4.937319780454559e-06\n",
      "select 2011 from TABLE_NAME_62 where 2008 = \"2r\" and 2012 = \"4r\"</s>\n",
      "5.29593563079834\n",
      "1199 7144 0.01750926673412323 0.0023769629187881947 9.86959712705652e-06 4.93479856352826e-06\n",
      "select count(RANK) from TABLE_NAME_39 where RANK = \"8\"</s>\n",
      "5.224110126495361\n",
      "1207 7144 -0.009705844335258007 0.0008603553869761527 9.864456609700726e-06 4.932228304850363e-06\n",
      "select TITLE from TABLE_29102100_1 where DIRECTED_BY = \"rich crell\" and WRITTEN_BY = \"dennis rinsler\"</s>\n",
      "5.3922038078308105\n",
      "1215 7144 0.0011636735871434212 0.0006731412722729146 9.859218112382117e-06 4.9296090561910585e-06\n",
      "select % similarity_to_c7orf38 from TABLE_26957063_3 where _percentage_identity_to_c7orf38 = \"81\"</s>\n",
      "4.856788635253906\n",
      "1223 7144 0.015565412119030952 0.004834500607103109 9.853881740614591e-06 4.9269408703072955e-06\n",
      "select OPPONENT from TABLE_NAME_6 where WEEK = \"2\"</s>\n",
      "5.074861526489258\n",
      "1231 7144 -0.026003720238804817 0.0032801292836666107 9.848447601883436e-06 4.924223800941718e-06\n",
      "select SCHOOL_CLUB_TEAM from TABLE_NAME_66 where PLAYER = \"SHAWN RESPRET\"</s>\n",
      "5.3126749992370605\n",
      "1239 7144 -0.021002117544412613 0.002481186995282769 9.842915805643156e-06 4.921457902821578e-06\n",
      "select PICK from TABLE_NAME_96 where ROUND > 2 and POSITION > OGRACIAL GROUND and PLAYER = REGGIE MACKENZIE</s>\n",
      "5.703678131103516\n",
      "1247 7144 0.0010206332663074136 0.0007935995818115771 9.837286463315284e-06 4.918643231657642e-06\n",
      "select QUAL_1 from TABLE_NAME_93 where NAME = \"alex tagliani\"</s>\n",
      "5.539216995239258\n",
      "1255 7144 0.011839032173156738 0.0011131191859021783 9.83155968828612e-06 4.91577984414306e-06\n",
      "select h_a_n from TABLE_NAME_64 where RECORD = \"20-48\"</s>\n",
      "5.041611194610596\n",
      "1263 7144 0.015084419399499893 0.0021588089875876904 9.825735595904462e-06 4.912867797952231e-06\n",
      "select DISTANCE from TABLE_NAME_73 where TIME = \"2:20.00\"</s>\n",
      "5.889540195465088\n",
      "1271 7144 -0.013528943061828613 0.001402081805281341 9.819814303479268e-06 4.909907151739634e-06\n",
      "select LONG from TABLE_NAME_28 where GAIN < 0</s>\n",
      "5.248817443847656\n",
      "1279 7144 -0.016958491876721382 0.0017730213003233075 9.813795930277306e-06 4.906897965138653e-06\n",
      "select COUNTRY from TABLE_2351952_1 where NAMES = \"801-812\"</s>\n",
      "5.746347904205322\n",
      "1287 7144 0.0062704249285161495 0.0015260902000591159 9.807680597520746e-06 4.903840298760373e-06\n",
      "select SCHOOL_CLUB_TEAM from TABLE_NAME_31 where PICK = \"33\"</s>\n",
      "5.081367492675781\n",
      "1295 7144 0.004802069626748562 0.0007537647034041584 9.801468428384716e-06 4.900734214192358e-06\n",
      "select AWAY_TEAM from TABLE_29728592_2 where COMPETITION = \"republic of srpska football day\" and HOME_TEAM = \"u 14 republic of srpska\"</s>\n",
      "4.847465991973877\n",
      "1303 7144 -0.014691594988107681 0.0013455607695505023 9.79515954799483e-06 4.897579773997415e-06\n",
      "select VISITOR from TABLE_NAME_63 where HOME = \"toronto maple leafs\" and SCORE = \"4–1\"</s>\n",
      "5.564592361450195\n",
      "1311 7144 -0.01044213306158781 0.0011323589133098722 9.788754083424654e-06 4.894377041712327e-06\n",
      "select REGULAR SEASON WINNER from TABLE_21091982_3 where CONFERENCE = \"Ivy League\"</s>\n",
      "6.22038459777832\n",
      "1319 7144 0.00868916418403387 0.0007460540509782732 9.782252163693159e-06 4.8911260818465795e-06\n",
      "select RESULT from TABLE_NAME_46 where NOMINEE = \"BRENT SPINNER\"</s>\n",
      "5.471485614776611\n",
      "1327 7144 0.00709870271384716 0.0009368231403641403 9.775653919762117e-06 4.887826959881058e-06\n",
      "select AWAY_TEAM from TABLE_NAME_91 where DATE = \"10 FEBRUARY 1951\" and HOME_TEAM = \"MANCHESTER UNITED\"</s>\n",
      "5.8112921714782715\n",
      "1335 7144 -0.002556699560955167 0.00026515801437199116 9.768959484533461e-06 4.884479742266731e-06\n",
      "select MAX(POINTS) from TABLE_NAME_93 where YEAR = 1983</s>\n",
      "5.92972469329834\n",
      "1343 7144 -0.007236538454890251 0.00044035419705323875 9.762168992846615e-06 4.8810844964233075e-06\n",
      "select DRAWS from TABLE_NAME_74 where ARTIST = \"Maggie Toal\"</s>\n",
      "5.578978538513184\n",
      "1351 7144 -0.010467326268553734 0.0019613674376159906 9.755282581475769e-06 4.8776412907378845e-06\n",
      "select COUNT(crowd) from TABLE_NAME_42 where HOME_TEAM = \"ESSENDON\"</s>\n",
      "5.651727676391602\n",
      "1359 7144 0.022102298215031624 0.0025687238667160273 9.748300389127132e-06 4.874150194563566e-06\n",
      "select AWAY from TABLE_NAME_71 where HOME = \"toronto downtown dingos\"</s>\n",
      "5.489588260650635\n",
      "1367 7144 0.0269163828343153 0.0042020464316010475 9.741222556436132e-06 4.870611278218066e-06\n",
      "select D_50 from TABLE_NAME_84 where D_41 = \"D 50\"</s>\n",
      "4.961786270141602\n",
      "1375 7144 0.008952613919973373 0.0012617635075002909 9.734049225964591e-06 4.867024612982296e-06\n",
      "select LOSER from TABLE_NAME_89 where DATE = \"october 18\" and LOCATION = \"schaefer stadium\"</s>\n",
      "5.588557720184326\n",
      "1383 7144 -0.010734844021499157 0.0016048959223553538 9.726780542197845e-06 4.863390271098922e-06\n",
      "select GOALS_FOR from TABLE_NAME_26 where LOST = \"25\"</s>\n",
      "6.146129131317139\n",
      "1391 7144 -0.013334816321730614 0.0012540273601189256 9.719416651541839e-06 4.859708325770919e-06\n",
      "select NAME from TABLE_NAME_32 where CAPS > 17 and RANK > 17</s>\n",
      "5.87609338760376\n",
      "1399 7144 -0.008912649936974049 0.0013063709484413266 9.711957702320176e-06 4.855978851160088e-06\n",
      "select DUST_OR_DEBRIS_LOCATION_AUK from TABLE_2296507_1 where STAR = \"HD 69830\"</s>\n",
      "4.831767559051514\n",
      "1407 7144 0.01177691575139761 0.0009837276302278042 9.704403844771128e-06 4.852201922385564e-06\n",
      "select RANK from TABLE_NAME_82 where U_17_GOALS = \"7\" and PLAYER = \"Cesc Fàbregas\"</s>\n",
      "5.369369029998779\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1415 7144 0.022002296522259712 0.0027758278883993626 9.696755231044618e-06 4.848377615522309e-06\n",
      "select DATE from TABLE_NAME_69 where TOURNAMENT = \"MILAN, itALY\"</s>\n",
      "5.360114097595215\n",
      "1423 7144 0.010338093154132366 0.0009449781500734389 9.689012015199147e-06 4.844506007599573e-06\n",
      "select MALE_LIFE_EXPERIENCE from TABLE_2701625_1 where COUNTRY = \"pakistan\"</s>\n",
      "4.851866245269775\n",
      "1431 7144 0.009558621793985367 0.005244977306574583 9.681174353198687e-06 4.8405871765993435e-06\n",
      "select COMPOSER_S from TABLE_NAME_85 where LENGTH = \"6:24\"</s>\n",
      "5.882352828979492\n",
      "1439 7144 -0.04882045462727547 0.010995034128427505 9.673242402909555e-06 4.836621201454778e-06\n",
      "select INSTITUTION from TABLE_NAME_78 where LOCATION = \"AKRON\"</s>\n",
      "5.936544895172119\n",
      "1447 7144 -0.06635109335184097 0.020015275105834007 9.665216324097222e-06 4.832608162048611e-06\n",
      "select PLACE from TABLE_NAME_6 where MONEY___$__ = \"73\" and PLAYER = \"ARCHARIC COMPSTON\"</s>\n",
      "5.788747310638428\n",
      "1455 7144 -0.06763730198144913 0.02119455300271511 9.657096278423093e-06 4.828548139211546e-06\n",
      "select max(phone_number) from employees where salary > 12000</s>\n",
      "5.270446300506592\n",
      "1463 7144 -0.03131752088665962 0.006323354784399271 9.648882429441258e-06 4.824441214720629e-06\n",
      "select FOUNDED from TABLE_2589963_1 where GREEN_AND_WALKING_COUNTRIES = \"GOLDEN_STATE_STATE_WALKING_COUNTRIES\"</s>\n",
      "5.685723304748535\n",
      "1471 7144 -0.015214730985462666 0.0017394238384440541 9.640574942595195e-06 4.820287471297598e-06\n",
      "select GROUND from TABLE_NAME_59 where AWAY_TEAM = \"Sydney\"</s>\n",
      "5.933882713317871\n",
      "1479 7144 0.016769297420978546 0.0014481850666925311 9.632173985214438e-06 4.816086992607219e-06\n",
      "select ENGINE from TABLE_NAME_24 where CHASSIS = \"LOTUS 44 F2\"</s>\n",
      "5.610810279846191\n",
      "1487 7144 0.02570510283112526 0.003425896866247058 9.623679726511204e-06 4.811839863255602e-06\n",
      "select LEAGUE_FROM from TABLE_NAME_69 where NHL_TEAM = \"ATLANTA THRAHERS\"</s>\n",
      "5.596269130706787\n",
      "1495 7144 0.03993202745914459 0.007116200868040323 9.615092337576987e-06 4.8075461687884935e-06\n",
      "select RACES from TABLE_NAME_15 where WINS > 8</s>\n",
      "5.928467750549316\n",
      "1503 7144 0.04490847513079643 0.010119069367647171 9.606411991379113e-06 4.803205995689557e-06\n",
      "select sprint_classification FROM table_23944514_15 WHERE aggressive_rider = \"michael barry\"</s>\n",
      "4.232079982757568\n",
      "1511 7144 0.01490038726478815 0.0013756321277469397 9.597638862757255e-06 4.7988194313786275e-06\n",
      "select YEAR from TABLE_NAME_25 where RESULT = \"3\" and VENUE = \"JARAMA\"</s>\n",
      "5.611158847808838\n",
      "1519 7144 -0.023641075938940048 0.0030263594817370176 9.588773128419907e-06 4.794386564209953e-06\n",
      "select F/Laps from TABLE_NAME_77 where POINTS = \"27\"</s>\n",
      "5.161473274230957\n",
      "1527 7144 -0.04130471870303154 0.009189932607114315 9.579814966940833e-06 4.7899074834704165e-06\n",
      "select PART_NUMBER_S from TABLE_NAME_52 where turbo = \"9/11\" and frequency = \"1.5 ghz\"</s>\n",
      "5.454751968383789\n",
      "1535 7144 -0.04644777625799179 0.009662670083343983 9.570764558755466e-06 4.785382279377733e-06\n",
      "select DATE from TABLE_NAME_37 where AGAINST < 21 and STATUS = \"SECOND TEST\"</s>\n",
      "5.539583206176758\n",
      "1543 7144 -0.023476071655750275 0.004121775738894939 9.561622086157273e-06 4.780811043078636e-06\n",
      "select max(episode__number) from TABLE_2501754_4 where EPISODE__NUMBER = \"LEAST\"</s>\n",
      "4.886366844177246\n",
      "1551 7144 -0.0037742380518466234 0.0011016621720045805 9.552387733294081e-06 4.7761938666470405e-06\n",
      "select UNIVERSITY from TABLE_29538735_1 where _PERCENTAGE_ELECTRICITY_REDUCTION = \"10%\"</s>\n",
      "5.446641445159912\n",
      "1559 7144 0.019289080053567886 0.0024196046870201826 9.543061686164374e-06 4.771530843082187e-06\n",
      "select TRIES_AGAINST from TABLE_NAME_9 where POINTS_AGAINST > 109</s>\n",
      "5.3446245193481445\n",
      "1567 7144 0.030642345547676086 0.004715208895504475 9.533644132613542e-06 4.766822066306771e-06\n",
      "select avg(pick) from TABLE_NAME_29 where ROUND = \"3\" and COLLEGE = \"PRINCE\"</s>\n",
      "5.4364914894104\n",
      "1575 7144 0.029261432588100433 0.004630088806152344 9.524135262330098e-06 4.762067631165049e-06\n",
      "select METHOD from TABLE_NAME_63 where TIME = \"1:46\"</s>\n",
      "5.005608558654785\n",
      "1583 7144 0.018063785508275032 0.0020777476020157337 9.514535266841863e-06 4.757267633420931e-06\n",
      "select APRIL from TABLE_NAME_50 where GAME = \"APRIL\" and POINTS = \"95\" and RECORD = \"41–25–10–3\"</s>\n",
      "5.631292343139648\n",
      "1591 7144 -0.013556151650846004 0.0018835236551240087 9.504844339512096e-06 4.752422169756048e-06\n",
      "select ATTENDANCE from TABLE_NAME_95 where OPPONENTS = \"HEARTS\"</s>\n",
      "5.924719333648682\n",
      "1599 7144 -0.02049446664750576 0.002272272016853094 9.495062675535614e-06 4.747531337767807e-06\n",
      "select RECORD from TABLE_NAME_8 where SCORE = \"7 - 2\"</s>\n",
      "5.740438461303711\n",
      "1607 7144 -0.0027515259571373463 0.0016614357009530067 9.485190471934845e-06 4.7425952359674225e-06\n",
      "select COLLEGE from TABLE_NAME_42 where PICK__NUMBER = \"145\"</s>\n",
      "5.475483417510986\n",
      "1615 7144 -0.011398633942008018 0.0009435734828002751 9.475227927555873e-06 4.737613963777936e-06\n",
      "select PLACE_OF_BIRTH from TABLE_NAME_62 where ACQUIRED = \"2008\"</s>\n",
      "5.694368839263916\n",
      "1623 7144 0.010398007929325104 0.001114993472583592 9.465175243064428e-06 4.732587621532214e-06\n",
      "select PLACE from TABLE_NAME_72 where SCORE = \"67-67=134\"</s>\n",
      "5.849596977233887\n",
      "1631 7144 0.01570005528628826 0.0012054438702762127 9.45503262094184e-06 4.72751631047092e-06\n",
      "select COUNT(crowd) from TABLE_NAME_59 where AWAY_TEAM = \"OFFICE\"</s>\n",
      "6.032407283782959\n",
      "1639 7144 0.0006248209392651916 0.00032627020846121013 9.444800265480968e-06 4.722400132740484e-06\n",
      "select SECOND_MEMBER from TABLE_NAME_3 where FIRST_MEMBER = \"sir rowland hill\" and SECOND_PARTY = \"conservative\"</s>\n",
      "5.730431079864502\n",
      "1647 7144 -0.009726496413350105 0.0009419050184078515 9.434478382782075e-06 4.717239191391038e-06\n",
      "select GAME_SITE from TABLE_NAME_85 where WEEK < 11 and DATE < october 21, 1973</s>\n",
      "5.47959041595459\n",
      "1655 7144 -0.005702798720449209 0.0004524125251919031 9.424067180748692e-06 4.712033590374346e-06\n",
      "select SCORE from TABLE_NAME_17 where RECORD = \"73-83\"</s>\n",
      "5.085846424102783\n",
      "1663 7144 0.01662474311888218 0.0014805844984948635 9.413566869083417e-06 4.706783434541708e-06\n",
      "select CName from tryout where pos = \"mid\"</s>\n",
      "5.226778507232666\n",
      "1671 7144 0.02460620366036892 0.0033696999307721853 9.40297765928369e-06 4.701488829641845e-06\n",
      "select STATE_TERRITORY from TABLE_NAME_58 where HEIGHT = \"121 m\"</s>\n",
      "5.343153476715088\n",
      "1679 7144 0.010398461483418941 0.0007628804305568337 9.39229976463755e-06 4.696149882318775e-06\n",
      "select NUMBER_OF_CONTESTANTS from TABLE_NAME_81 where CYCLE = \"4\"</s>\n",
      "5.416485786437988\n",
      "1687 7144 -0.024639995768666267 0.0031555388122797012 9.381533400219319e-06 4.690766700109659e-06\n",
      "select MIN(pick__number) from TABLE_20649850_1 where COLLEGE = \"new mexico\"</s>\n",
      "4.7157464027404785\n",
      "1695 7144 -0.03320614621043205 0.005292722024023533 9.37067878288528e-06 4.68533939144264e-06\n",
      "select DATE from TABLE_NAME_44 where LOSS = \"ruffin (3-5)\"</s>\n",
      "5.303475856781006\n",
      "1703 7144 -0.024874715134501457 0.003286038525402546 9.359736131269312e-06 4.679868065634656e-06\n",
      "select BRONZE from TABLE_NAME_59 where PLACE = \"MELBOURNE\"</s>\n",
      "6.330538272857666\n",
      "1711 7144 -0.013079543597996235 0.0015647773398086429 9.348705665778479e-06 4.674352832889239e-06\n",
      "select STREAK from TABLE_NAME_1 where LOCATION_ATTENDANCE = \"staples center\" and SCORE = \"67–89\"</s>\n",
      "6.204545974731445\n",
      "1719 7144 0.029191292822360992 0.004772023297846317 9.337587608588588e-06 4.668793804294294e-06\n",
      "select count(no) from TABLE_1569625_1 where RUNNER_S__UP = \"HOWARD CLARK\"</s>\n",
      "4.40100622177124\n",
      "1727 7144 0.033467113971710205 0.00567823089659214 9.326382183639731e-06 4.663191091819866e-06\n",
      "select SCHOOL from TABLE_1984697_85 where CITY___town = \"NORTH MANCHESTER\"</s>\n",
      "5.990864276885986\n",
      "1735 7144 0.02165791764855385 0.002787158824503422 9.315089616631752e-06 4.657544808315876e-06\n",
      "select CIRCUIT from TABLE_25459168_2 where DATE = \"AUGUST 21\"</s>\n",
      "5.4555463790893555\n",
      "1743 7144 0.026533884927630424 0.005266379099339247 9.30371013501972e-06 4.65185506750986e-06\n",
      "select ELECTION_DATE from TABLE_1193568_1 where ELECTORATE = \"christchurch country\"</s>\n",
      "3.777634620666504\n",
      "1751 7144 -0.005830586422234774 0.0004707986081484705 9.292243968009332e-06 4.646121984004666e-06\n",
      "select DATE from TABLE_NAME_20 where COMPETITION = \"2002 AFRICAN CUP OF NATIONS\"</s>\n",
      "5.817237854003906\n",
      "1759 7144 -0.017928162589669228 0.0018408575560897589 9.280691346552308e-06 4.640345673276154e-06\n",
      "select EVENING_GOWN from TABLE_NAME_60 where AVERAGE > 8.984 and PRELIMINARY > 8.597</s>\n",
      "6.245542049407959\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1767 7144 -0.015607263892889023 0.0016024785581976175 9.269052503341737e-06 4.634526251670869e-06\n",
      "select COUNT(mountain_id) from mountains where camera_lens_id = \"ETHIOPIA\"</s>\n",
      "5.312885284423828\n",
      "1775 7144 -0.004942224361002445 0.0003870948567055166 9.257327672807383e-06 4.628663836403692e-06\n",
      "select BILLS_POINTS from TABLE_NAME_54 where DATE = \"NOVEMBER 11\"</s>\n",
      "6.130627632141113\n",
      "1783 7144 0.01442668680101633 0.001323877600952983 9.24551709111097e-06 4.622758545555485e-06\n",
      "select DATE from TABLE_NAME_66 where RESULT = \"W20-0\"</s>\n",
      "5.443122386932373\n",
      "1791 7144 0.028410406783223152 0.003852252848446369 9.233620996141421e-06 4.6168104980707105e-06\n",
      "select CLASS from TABLE_NAME_6 where QUANTITY = \"1\"</s>\n",
      "5.628842830657959\n",
      "1799 7144 0.012749232351779938 0.0008872277685441077 9.221639627510076e-06 4.610819813755038e-06\n",
      "select HOME from TABLE_NAME_31 where RECORD = \"3-2\"</s>\n",
      "5.485411643981934\n",
      "1807 7144 -0.0044966209679841995 0.000602404703386128 9.209573226545852e-06 4.604786613272926e-06\n",
      "select count(no_in_series) from TABLE_2226817_8 where NO_IN_SEASON = \"12\"</s>\n",
      "5.157860279083252\n",
      "1815 7144 -0.012080016545951366 0.0008683371124789119 9.197422036290386e-06 4.598711018145193e-06\n",
      "select Police_force FROM county_public_safety WHERE LOCATION = \"EAST SIDE\"</s>\n",
      "5.7677717208862305\n",
      "1823 7144 -0.01456679031252861 0.0012312913313508034 9.185186301493153e-06 4.592593150746576e-06\n",
      "select NUMBER_ from TABLE_NAME_72 where DESCRIPTION = \"pECKETT 0-4-0ST\"</s>\n",
      "5.5379252433776855\n",
      "1831 7144 0.005917781498283148 0.00041385003714822233 9.172866268606514e-06 4.586433134303257e-06\n",
      "select RANK from TABLE_NAME_91 where GOLD < 0</s>\n",
      "5.731602668762207\n",
      "1839 7144 0.008388033136725426 0.0007726745679974556 9.16046218578077e-06 4.580231092890385e-06\n",
      "select LOCATION_ATTENDANCE from TABLE_11960944_11 where HIGH_ASSISTS = \"BILLPUPS, STUCKEY (4)\"</s>\n",
      "5.9647088050842285\n",
      "1847 7144 0.004133991897106171 0.00026152958162128925 9.147974302859158e-06 4.573987151429579e-06\n",
      "select OVERALL from TABLE_NAME_64 where NATIONALITY = \"FINLAND\"</s>\n",
      "5.535691261291504\n",
      "1855 7144 -0.007731993682682514 0.0006397558026947081 9.13540287137281e-06 4.567701435686405e-06\n",
      "select SEASON_IN_Sanskrit from TABLE_NAME_34 where SEASON_IN_EAST_1 = \"MAY\"</s>\n",
      "5.167282581329346\n",
      "1863 7144 -0.009135334752500057 0.0005695658619515598 9.122748144535704e-06 4.561374072267852e-06\n",
      "select POSITION from TABLE_NAME_62 where LOST > 9 and DRAWN > 9</s>\n",
      "5.496738910675049\n",
      "1871 7144 -0.0012167564127594233 0.00017975791706703603 9.110010377239552e-06 4.555005188619776e-06\n",
      "select avg(AGE) from singer where COUNTRY = \"FRANCE\"</s>\n",
      "5.646761894226074\n",
      "1879 7144 0.007233606185764074 0.0006143823848105967 9.09718982604866e-06 4.54859491302433e-06\n",
      "select WEEKLY_RANK from TABLE_11178271_1 where AIR_DATE = \"NOVEMBER 12, 2007\"</s>\n",
      "5.515735626220703\n",
      "1887 7144 0.010310495272278786 0.0006156102754175663 9.084286749194783e-06 4.542143374597391e-06\n",
      "select DATE from TABLE_NAME_96 where AWAY_TEAM = \"GEELONG\"</s>\n",
      "5.666132926940918\n",
      "1895 7144 -0.0006233854219317436 0.0001578197261551395 9.071301406571893e-06 4.5356507032859465e-06\n",
      "select TITLE from TABLE_15431251_1 where NO_IN_SEASON = \"7\"</s>\n",
      "5.472372531890869\n",
      "1903 7144 -0.002377918688580394 0.0347367525100708 9.058234059730977e-06 4.529117029865488e-06\n",
      "select GAME from TABLE_NAME_89 where OPPONENT = \"WASHINGTON\"</s>\n",
      "6.475903034210205\n",
      "1911 7144 -0.015841156244277954 0.0014220403973013163 9.045084971874738e-06 4.522542485937369e-06\n",
      "select SCHOOL from TABLE_NAME_17 where EVENT = \"100 BACKstroke\"</s>\n",
      "5.118247985839844\n",
      "1919 7144 -0.006903607398271561 0.0006405598833225667 9.031854407852317e-06 4.5159272039261585e-06\n",
      "select count(france) from TABLE_NAME_37 where _PERCENTAGE_PEOPLE_UNDER_18__2005_ = \"SUB-SAHARAN AFRICA\"</s>\n",
      "5.094365119934082\n",
      "1927 7144 0.009000105783343315 0.0007752878591418266 9.018542634153944e-06 4.509271317076972e-06\n",
      "select RESULT from TABLE_NAME_58 where DATE = \"OCTOBER 8, 1985\"</s>\n",
      "5.5500359535217285\n",
      "1935 7144 0.016974976286292076 0.0017276644939556718 9.00514991890558e-06 4.50257495945279e-06\n",
      "select RECORD from TABLE_NAME_25 where OPPONENT = \"LA Rams\"</s>\n",
      "6.020169258117676\n",
      "1943 7144 0.003019894938915968 0.00028772378573194146 8.991676531863507e-06 4.495838265931754e-06\n",
      "select MAX(NO_IN_SEASON) from TABLE_28014096_1 where PRODUCTION_CODE = \"2t6206\"</s>\n",
      "5.272573947906494\n",
      "1951 7144 -0.003432764206081629 0.00036164134507998824 8.978122744408905e-06 4.4890613722044526e-06\n",
      "select AWAY_TEAM from TABLE_NAME_3 where VENUE = \"PRINCE PRINCE PARK\"</s>\n",
      "5.967184066772461\n",
      "1959 7144 -0.010017980821430683 0.0009845735039561987 8.964488829542377e-06 4.482244414771188e-06\n",
      "select HEADQUARTERS from TABLE_1682026_7 where COMPANY = \"HSBC\"</s>\n",
      "5.922702312469482\n",
      "1967 7144 0.0026707311626523733 0.00044626451563090086 8.950775061878453e-06 4.475387530939226e-06\n",
      "select TEAM from TABLE_NAME_5 where RANK < 377, COUNTRY < GERMANY and COUNTRY = \"GERMANY\"</s>\n",
      "5.583419322967529\n",
      "1975 7144 -0.000997471739538014 0.0004786945937667042 8.936981717640061e-06 4.4684908588200305e-06\n",
      "select LOCATION from TABLE_NAME_1 where YEAR = 1994 and COUNTRY = YEMEN</s>\n",
      "6.168649673461914\n",
      "1983 7144 0.005955602508038282 0.0004967580316588283 8.92310907465296e-06 4.46155453732648e-06\n",
      "select COMP from TABLE_NAME_84 where DATE = \"2004-10-17\"</s>\n",
      "5.309462547302246\n",
      "1991 7144 0.006443389691412449 0.0018016870599240065 8.90915741234015e-06 4.454578706170075e-06\n",
      "select Format FROM table_name_22 WHERE open_standard = \"no\" AND interactivity_support = \"no\" AND image_support = \"yes\" AND word_wrap_support = \"yes\"</s>\n",
      "4.28523588180542\n",
      "1999 7144 -0.021604476496577263 0.002574642188847065 8.895127011716234e-06 4.447563505858117e-06\n",
      "select SEASON_NO from TABLE_21781578_2 where DIRECTED_BY = \"JOHN DAVID COLES\"</s>\n",
      "5.325157165527344\n",
      "2007 7144 -0.020869916304945946 0.002452130662277341 8.881018155381766e-06 4.440509077690883e-06\n",
      "select fastest_laps FROM table_name_65 WHERE season = 2009 AND poles < 0</s>\n",
      "5.140619277954102\n",
      "2015 7144 -0.0003380880516488105 0.0031305821612477303 8.866831127517557e-06 4.4334155637587785e-06\n",
      "select 2011 FROM table_24027047_1 WHERE population_density___km²_2011 = \"684.37\"</s>\n",
      "5.022191047668457\n",
      "2023 7144 -0.0043328022584319115 0.00025868741795420647 8.852566213878947e-06 4.426283106939474e-06\n",
      "select DIES from TABLE_2930244_4 where NUMBER_OF_HURANTS = \"6\"</s>\n",
      "5.575573444366455\n",
      "2031 7144 0.016998572275042534 0.001601842581294477 8.838223701790057e-06 4.4191118508950286e-06\n",
      "select DISTRICT_ID from STORE_DISTRUCTURE where DISTRICT_NAME = \"Khanewal\"</s>\n",
      "5.993173122406006\n",
      "2039 7144 0.024178070947527885 0.0034301630221307278 8.823803880137993e-06 4.411901940068997e-06\n",
      "select SURFACE from TABLE_NAME_31 where OPPONENT = \"Jiří vaněk\"</s>\n",
      "4.961005210876465\n",
      "2047 7144 0.008342324756085873 0.0007518018828704953 8.809307039367035e-06 4.4046535196835175e-06\n",
      "select FIRST_APPEARANCE from TABLE_NAME_26 where portrayed_BY = \"CHAD WILLIAMS\"</s>\n",
      "5.847836971282959\n",
      "2055 7144 -0.007731928955763578 0.0011371421860530972 8.794733471472778e-06 4.397366735736389e-06\n",
      "select TITLE from TABLE_2182654_3 where DIRECTED_BY = \"MIGUEL ARTETA\"</s>\n",
      "5.6510114669799805\n",
      "2063 7144 0.0014541416894644499 0.0025994309689849615 8.780083469996264e-06 4.390041734998132e-06\n",
      "select ENDS from TABLE_25107064_2 where Blank_ends = \"0\"</s>\n",
      "4.614048957824707\n",
      "2071 7144 -0.03157013654708862 0.005767236463725567 8.765357330018056e-06 4.382678665009028e-06\n",
      "select RATING from TABLE_25391981_20 where EPISODE = \"PART 1\"</s>\n",
      "5.720459461212158\n",
      "2079 7144 -0.026398196816444397 0.003725628135725856 8.750555348152299e-06 4.3752776740761495e-06\n",
      "select OTHERS from TABLE_NAME_33 where MARTIN = \"9%\" and Source = \"OMG/NEWS\"</s>\n",
      "4.957484245300293\n",
      "2087 7144 -0.00834000576287508 0.0010391660034656525 8.73567782254075e-06 4.367838911270375e-06\n",
      "select COUNT(*) from customers where id = \"TOTAL\"</s>\n",
      "5.153680801391602\n",
      "2095 7144 0.014038017950952053 0.001548781874589622 8.720725052846766e-06 4.360362526423383e-06\n",
      "select RESULT from TABLE_14966537_1 where OPPONENT = \"CHICAGO BAREAS\"</s>\n",
      "5.889638423919678\n",
      "2103 7144 0.022600889205932617 0.002539826789870858 8.705697340249275e-06 4.352848670124637e-06\n",
      "select COUNTRY from TABLE_NAME_74 where PLAYER = \"nick faldo\"</s>\n",
      "5.368343830108643\n",
      "2111 7144 0.03008289635181427 0.004210391081869602 8.690594987436705e-06 4.345297493718352e-06\n",
      "select SURFACE from TABLE_NAME_15 where DATE = \"JULY 15, 1984\"</s>\n",
      "5.448113441467285\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2119 7144 0.010231131687760353 0.000676245370414108 8.675418298600884e-06 4.337709149300442e-06\n",
      "select OBAMA_NUMBER from TABLE_20799905_1 where COUNTY = \"LAKE COUNTY\"</s>\n",
      "5.362421035766602\n",
      "2127 7144 -0.008723954670131207 0.0007445139926858246 8.660167579430926e-06 4.330083789715463e-06\n",
      "select RACE from TABLE_NAME_3 where DATE = \"10/10/2008\"</s>\n",
      "5.42064905166626\n",
      "2135 7144 -0.016770772635936737 0.0015445343451574445 8.644843137107058e-06 4.322421568553529e-06\n",
      "select AWAY_TEAM from TABLE_NAME_21 where CROWD > 6,000</s>\n",
      "6.053621292114258\n",
      "2143 7144 -0.015165111981332302 0.0013569002039730549 8.629445280294445e-06 4.314722640147222e-06\n",
      "select DEPARTURE from TABLE_NAME_66 where STATION_NAME = \"Rajendranagar Terminal\"</s>\n",
      "5.984164714813232\n",
      "2151 7144 -0.011036969721317291 0.0012958166189491749 8.613974319136959e-06 4.3069871595684795e-06\n",
      "select MANNER_OF_DEPARTURE from TABLE_27495117_3 where DATE_OF_APPOINTMENT = \"21 september 2010\"</s>\n",
      "5.1301774978637695\n",
      "2159 7144 0.014538797549903393 0.0014261899050325155 8.598430565250953e-06 4.299215282625477e-06\n",
      "select Apartments from Apartments whereapt_type_code = \"apt_type_code\" and room_count = \"3\"</s>\n",
      "5.17596435546875\n",
      "2167 7144 0.024310994893312454 0.0030539161525666714 8.582814331718961e-06 4.291407165859481e-06\n",
      "select DIVISION from TABLE_1046170_5 where USOPEN_CUP = \"2003\"</s>\n",
      "5.361998081207275\n",
      "2175 7144 0.02458026073873043 0.0034455626737326384 8.567125933083415e-06 4.283562966541708e-06\n",
      "select MIN(margin_of_victory) from TABLE_NAME_34 where DATE = \"15 JUL 2007\"</s>\n",
      "5.807149887084961\n",
      "2183 7144 0.018382450565695763 0.004197393544018269 8.551365685340285e-06 4.275682842670143e-06\n",
      "select drawn FROM table_name_81 WHERE drawn < 0</s>\n",
      "3.9187769889831543\n",
      "2191 7144 -0.009626831859350204 0.0008516961825080216 8.535533905932739e-06 4.267766952966369e-06\n",
      "select FREQUENCY from TABLE_NAME_5 where CALLSIGN = \"2CFM\"</s>\n",
      "5.900415420532227\n",
      "2199 7144 -0.018183313310146332 0.001839827629737556 8.519630913744726e-06 4.259815456872363e-06\n",
      "select YEAR from TABLE_NAME_82 where MATCH = \"63\"</s>\n",
      "5.298587799072266\n",
      "2207 7144 -0.017963523045182228 0.0016096209874376655 8.503657029094569e-06 4.2518285145472845e-06\n",
      "select FIN_POS from TABLE_17244483_1 where DRIVER = \"Darren Manning\"</s>\n",
      "5.386806011199951\n",
      "2215 7144 -0.0013794563710689545 0.0006101173930801451 8.487612573728513e-06 4.243806286864257e-06\n",
      "select DATE from TABLE_11960407_5 where GAME = \"GAME 50\"</s>\n",
      "5.432788848876953\n",
      "2223 7144 0.004684881307184696 0.00027932776720263064 8.47149787081423e-06 4.235748935407115e-06\n",
      "select AWAY_TEAM from TABLE_NAME_29 where AWAY_TEAM = \"CARLTON\"</s>\n",
      "5.375504016876221\n",
      "2231 7144 0.013247030787169933 0.0011339557822793722 8.455313244934324e-06 4.227656622467162e-06\n",
      "select PARTY from TABLE_1341522_24 where DISTRICT = \"MASSACHUSETTS4\"</s>\n",
      "5.793990612030029\n",
      "2239 7144 0.007300585974007845 0.0006400247220881283 8.439059022079789e-06 4.219529511039894e-06\n",
      "select OVERALL from TABLE_NAME_30 where POSITION = \"COUNTRICK\" and PICK__NUMBER < 16</s>\n",
      "5.689102649688721\n",
      "2247 7144 0.0009028376662172377 0.00035994561039842665 8.422735529643445e-06 4.211367764821722e-06\n",
      "select PLAYER from TABLE_NAME_73 where COLLEGE = \"APACHIAN STATE\"</s>\n",
      "5.63369083404541\n",
      "2255 7144 -0.01555116567760706 0.001697252388112247 8.406343096413333e-06 4.203171548206666e-06\n",
      "select RANK from TABLE_NAME_29 where FA = \"FA\"</s>\n",
      "5.66241979598999\n",
      "2263 7144 -0.0220596045255661 0.003235095413401723 8.389882052566106e-06 4.194941026283053e-06\n",
      "select DATE from TABLE_1140090_6 where WINNING_DRIVER = \"Chris Amon\"</s>\n",
      "5.487020015716553\n",
      "2271 7144 -0.008482183329761028 0.0009276582859456539 8.373352729660373e-06 4.186676364830187e-06\n",
      "select AREA from TABLE_1982739_2 where ENGLISH_NAME = \"NANQIAPO DISTRICT\"</s>\n",
      "6.422829627990723\n",
      "2279 7144 0.002552520949393511 0.0006208918639458716 8.35675546063002e-06 4.17837773031501e-06\n",
      "select ENGINE_NAME from TABLE_NAME_42 where MAX_TORQUE_AT_RPM = \"n·m ( lbf·ft ) @ 3,800\"</s>\n",
      "5.118289470672607\n",
      "2287 7144 0.012669217772781849 0.0008778321207500994 8.340090579777507e-06 4.170045289888754e-06\n",
      "select CATALOG from TABLE_NAME_58 where LABEL = \"rca\"</s>\n",
      "6.425224304199219\n",
      "2295 7144 0.01587875373661518 0.0017762671923264861 8.32335842276713e-06 4.161679211383565e-06\n",
      "select count(no_in_series) from TABLE_18335117_5 where NO_OVERALL = \"38\"</s>\n",
      "4.426743507385254\n",
      "2303 7144 0.00414106622338295 0.00037155405152589083 8.30655932661826e-06 4.15327966330913e-06\n",
      "select max(fall_09) from TABLE_15055594_6 where Fall 06 = 3821</s>\n",
      "5.1735615730285645\n",
      "2311 7144 -0.0057996767573058605 0.00040289422031491995 8.289693629698564e-06 4.144846814849282e-06\n",
      "select RES from TABLE_NAME_7 where RECORD = \"3–1\"</s>\n",
      "4.9578399658203125\n",
      "2319 7144 -0.0083326930180192 0.0008215464185923338 8.272761671717178e-06 4.136380835858589e-06\n",
      "select YEAR from TABLE where TIES = \"1\"</s>\n",
      "5.881969928741455\n",
      "2327 7144 -0.011613232083618641 0.0008962361607700586 8.255763793717868e-06 4.127881896858934e-06\n",
      "select LOCATION from TABLE where TRAIN_STATIONS = \"MOST\"</s>\n",
      "6.000301361083984\n",
      "2335 7144 0.00676212040707469 0.0009428223711438477 8.238700338072167e-06 4.1193501690360835e-06\n",
      "select max(ep) from TABLE_20967430_2 where ORIGINAL_AIR_DATE = \"NOVEMBER 29, 1985\"</s>\n",
      "4.508133411407471\n",
      "2343 7144 0.006586496718227863 0.0005513504729606211 8.221571648472473e-06 4.1107858242362366e-06\n",
      "select OPPONENT from TABLE_NAME_52 where WEEK = \"6\"</s>\n",
      "5.953267574310303\n",
      "2351 7144 0.004288190044462681 0.00033725876710377634 8.204378069925121e-06 4.102189034962561e-06\n",
      "select PLACE from TABLE_NAME_86 where COUNTRY = \"australia\" and PLAYER = \"craig parry\"</s>\n",
      "5.887331962585449\n",
      "2359 7144 -0.006502519827336073 0.0003541608457453549 8.18711994874345e-06 4.093559974371725e-06\n",
      "select DIRECTED_BY from TABLE_17641206_4 where WRITTEN_BY = \"JOHN SULLIVAN\"</s>\n",
      "5.487724781036377\n",
      "2367 7144 -0.006185681093484163 0.0003440635628066957 8.16979763254081e-06 4.084898816270405e-06\n",
      "select count(touchdowns) from TABLE_NAME_53 where EXTRA_POINTS < 0 and FIELD_GOALS < 0</s>\n",
      "5.810417652130127\n",
      "2375 7144 0.0012371719349175692 0.00021190219558775425 8.15241147022357e-06 4.076205735111785e-06\n",
      "select MAX(PLAYED) from TABLE_14489821_1 where TEAM_MATCHES = \"PLAYED\"</s>\n",
      "5.352804183959961\n",
      "2383 7144 0.006242964882403612 0.0005247138906270266 8.13496181198409e-06 4.067480905992045e-06\n",
      "select YEAR from TABLE where 2000 = \"no\"</s>\n",
      "5.184416770935059\n",
      "2391 7144 0.002835601568222046 0.0003093583509325981 8.117449009293668e-06 4.058724504646834e-06\n",
      "select PARTNER from TABLE_NAME_94 where SCORE_IN_FINAL = \"6–1, 6–2\"</s>\n",
      "5.777475357055664\n",
      "2399 7144 0.0014529549516737461 0.00020870091975666583 8.099873414895453e-06 4.049936707447727e-06\n",
      "select DATE from TABLE_NAME_51 where STATUS = \"AGAINST\" and AGAINST = \"22\"</s>\n",
      "5.663441181182861\n",
      "2407 7144 -0.006994625087827444 0.0004647332534659654 8.08223538279735e-06 4.041117691398675e-06\n",
      "select SCORE from TABLE_NAME_75 where OPPONENT = \"TIGERS\" and RECORD = \"78-64\"</s>\n",
      "6.3292365074157715\n",
      "2415 7144 -0.0031377323903143406 0.0004172523913439363 8.064535268264883e-06 4.032267634132442e-06\n",
      "select CAPACITY from TABLE_NAME_69 where HEAD_COACH = \"RAINER KRAFT\"</s>\n",
      "6.128314018249512\n",
      "2423 7144 0.0034395400434732437 0.00019809261721093208 8.046773427814043e-06 4.023386713907021e-06\n",
      "select CALLSIGN from TABLE_NAME_34 where BRANDING = \"MOM'S RADIO 101.9 ZAMBOANGA\"</s>\n",
      "5.587402820587158\n",
      "2431 7144 0.0005290331318974495 0.00023192328808363527 8.0289502192041e-06 4.01447510960205e-06\n",
      "select LEAGUE from TABLE_NAME_20 where TOTAL <143</s>\n",
      "5.700315952301025\n",
      "2439 7144 0.0007526903646066785 0.0001364128984278068 8.011066001430412e-06 4.005533000715206e-06\n",
      "select SCORE from TABLE_NAME_29 where DECISION = \"JOHNSON\"</s>\n",
      "5.1731133460998535\n",
      "2447 7144 0.0006356602534651756 0.0003118737367913127 7.993121134717177e-06 3.996560567358589e-06\n",
      "select OPENED_IN from TABLE_NAME_93 where RIDE_NAME = \"Tiny Truckers\" and MODE = \"Convoy\" and TYPE = \"Tiny Truckers\"</s>\n",
      "4.727443695068359\n",
      "2455 7144 -0.0029864786192774773 0.00037023783079348505 7.975115980510187e-06 3.987557990255093e-06\n",
      "select SCORE from TABLE_11960610_10 where LOCATION_ATTENDANCE = \"UNITED CENTER 22,097\"</s>\n",
      "5.9947590827941895\n",
      "2463 7144 -0.0008695429423823953 0.0001615638320799917 7.957050901469544e-06 3.978525450734772e-06\n",
      "select max(CAPS) from TABLE_NAME_55 where POSITION = \"LOCK\" and CLUB_PROSPERCE = \"VICENZA rANGERS\"</s>\n",
      "5.566850662231445\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2471 7144 0.009253652766346931 0.0009804638102650642 7.938926261462366e-06 3.969463130731183e-06\n",
      "select cardinalatial_title from TABLE_NAME_40 where elector = \"luccan\" and elevated = \"december 18, 1182\"</s>\n",
      "4.73054313659668\n",
      "2479 7144 -0.0017807321855798364 0.00042110608774237335 7.920742425555436e-06 3.960371212777718e-06\n",
      "select max(population__2002_census_data_) from TABLE_22854436_1 where 2002 census_data = \"population__2002_census_data_\"</s>\n",
      "4.464055061340332\n",
      "2487 7144 -0.008563969284296036 0.0006649757269769907 7.902499760007867e-06 3.951249880003934e-06\n",
      "select POSITION from TABLE_11803648_20 where OVERALL = \"126\"</s>\n",
      "5.25340461730957\n",
      "2495 7144 0.0031870061066001654 0.0003407715994399041 7.884198632263725e-06 3.942099316131862e-06\n",
      "select RESTYPE DESCRIPTION from TABLE_NAME where RESTYPE NAME = \"SANDWICH\"</s>\n",
      "5.651818752288818\n"
     ]
    }
   ],
   "source": [
    "for i, data in enumerate(loader):\n",
    "    #生成数据\n",
    "    (generate, generate_mask, prob_old, prob_ref, value_old,\n",
    "     reward) = get_batch(**data)\n",
    "\n",
    "    do_step = (i + 1) % 8 == 0\n",
    "\n",
    "    #训练\n",
    "    loss_actor, loss_critic = train(generate, generate_mask, prob_old,\n",
    "                                    prob_ref, value_old, reward, do_step)\n",
    "\n",
    "    if do_step:\n",
    "        lr_actor = optimizer_actor.param_groups[0]['lr']\n",
    "        lr_critic = optimizer_critic.param_groups[0]['lr']\n",
    "        print(i, len(loader), loss_actor, loss_critic, lr_actor, lr_critic)\n",
    "        print(tokenizer.decode(generate[0, 256:]))\n",
    "        print(reward[0].item())\n",
    "\n",
    "    if i == 2500:\n",
    "        break\n",
    "\n",
    "lora.merge(model_actor)\n",
    "model_actor.save_pretrained('model/rlhf')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cuda117]",
   "language": "python",
   "name": "conda-env-cuda117-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
